feat(normalizer): add new configurations for text normalization

entelecheia · Jul 23, 2023 · ac53a45 · ac53a45
1 parent 53908dd
commit ac53a45
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 0 deletions.
diff --git a/src/corprep/conf/normalizer/__init__.yaml b/src/corprep/conf/normalizer/__init__.yaml
@@ -0,0 +1,8 @@
+defaults:
+  - ftfy: __init__
+  - spaces: __init__
+  - special_characters: __init__
+
+_target_: corprep.tokenizer.normalizer.Normalizer
+hanja2hangle: false
+num_repeats: 2
diff --git a/src/corprep/conf/normalizer/formal_en.yaml b/src/corprep/conf/normalizer/formal_en.yaml
@@ -0,0 +1,5 @@
+defaults:
+  - __init__
+
+ftfy:
+  unescape_html: "auto"
diff --git a/src/corprep/conf/normalizer/formal_en_parantheses.yaml b/src/corprep/conf/normalizer/formal_en_parantheses.yaml
@@ -0,0 +1,5 @@
+defaults:
+  - __init__
+
+special_characters:
+  regular_parentheses_only: true
diff --git a/src/corprep/conf/normalizer/formal_ko.yaml b/src/corprep/conf/normalizer/formal_ko.yaml
@@ -0,0 +1,4 @@
+defaults:
+  - __init__
+
+hanja2hangle: true
diff --git a/src/corprep/conf/normalizer/informal_ko.yaml b/src/corprep/conf/normalizer/informal_ko.yaml
@@ -0,0 +1,5 @@
+defaults:
+  - formal_ko
+
+special_characters:
+  fix_emoticons: true