diff --git a/Jenkinsfile b/Jenkinsfile index 51ce37a10..2f9ca394d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -176,7 +176,7 @@ pipeline { } } - stage('L0: Create FR TN/ITN & VI ITN & HU TN & IT TN') { + stage('L0: Create FR TN/ITN & VI TN/ITN & HU TN & IT TN') { when { anyOf { branch 'main' @@ -200,6 +200,11 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=vi --text="một ngàn " --cache_dir ${VI_TN_CACHE}' } } + stage('L0: VI TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=vi --text="100" --cache_dir ${VI_TN_CACHE}' + } + } stage('L0: HU TN grammars') { steps { sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hu --text="100" --cache_dir ${HU_TN_CACHE}' diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index 82f8f43d2..329b28338 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -174,6 +174,9 @@ def __init__( elif lang == 'ja': from nemo_text_processing.text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'vi': + from nemo_text_processing.text_normalization.vi.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.vi.verbalizers.verbalize_final import VerbalizeFinalFst else: raise NotImplementedError(f"Language {lang} has not been supported yet.") diff --git a/nemo_text_processing/text_normalization/vi/__init__.py b/nemo_text_processing/text_normalization/vi/__init__.py new file mode 100644 index 000000000..bc443be41 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/__init__.py b/nemo_text_processing/text_normalization/vi/data/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/__init__.py b/nemo_text_processing/text_normalization/vi/data/numbers/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/digit.tsv new file mode 100644 index 000000000..573c20bd4 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +1 một +2 hai +3 ba +4 bốn +5 năm +6 sáu +7 bảy +8 tám +9 chín \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/digit_special.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/digit_special.tsv new file mode 100644 index 000000000..919baaf6e --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/digit_special.tsv @@ -0,0 +1,3 @@ +1 một mốt +4 bốn tư +5 năm lăm \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/magnitudes.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/magnitudes.tsv new file mode 100644 index 000000000..c8a08083c --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/magnitudes.tsv @@ -0,0 +1,5 @@ +thousand nghìn +million triệu +billion tỷ +hundred trăm +linh linh \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/teen.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/teen.tsv new file mode 100644 index 000000000..8d99f8a69 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +10 mười +11 mười một +12 mười hai +13 mười ba +14 mười bốn +15 mười lăm +16 mười sáu +17 mười bảy +18 mười tám +19 mười chín \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/ties.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/ties.tsv new file mode 100644 index 000000000..da88b8ab8 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/ties.tsv @@ -0,0 +1,8 @@ +2 hai mươi +3 ba mươi +4 bốn mươi +5 năm mươi +6 sáu mươi +7 bảy mươi +8 tám mươi +9 chín mươi \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/zero.tsv new file mode 100644 index 000000000..df062e38c --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/zero.tsv @@ -0,0 +1 @@ +0 không \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/whitelist.tsv b/nemo_text_processing/text_normalization/vi/data/whitelist.tsv new file mode 100644 index 000000000..e69de29bb diff --git a/nemo_text_processing/text_normalization/vi/taggers/__init__.py b/nemo_text_processing/text_normalization/vi/taggers/__init__.py new file mode 100644 index 000000000..bc443be41 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/taggers/cardinal.py b/nemo_text_processing/text_normalization/vi/taggers/cardinal.py new file mode 100644 index 000000000..fa0f04fad --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/cardinal.py @@ -0,0 +1,181 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, insert_space +from nemo_text_processing.text_normalization.vi.utils import get_abs_path + + +class CardinalFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + resources = { + 'zero': pynini.string_file(get_abs_path("data/numbers/zero.tsv")), + 'digit': pynini.string_file(get_abs_path("data/numbers/digit.tsv")), + 'teen': pynini.string_file(get_abs_path("data/numbers/teen.tsv")), + 'ties': pynini.string_file(get_abs_path("data/numbers/ties.tsv")), + } + self.zero, self.digit, self.teen, self.ties = resources.values() + + with open(get_abs_path("data/numbers/magnitudes.tsv"), 'r', encoding='utf-8') as f: + self.magnitudes = {parts[0]: parts[1] for line in f if len(parts := line.strip().split('\t')) == 2} + + with open(get_abs_path("data/numbers/digit_special.tsv"), 'r', encoding='utf-8') as f: + special = { + parts[0]: {'std': parts[1], 'alt': parts[2]} + for line in f + if len(parts := line.strip().split('\t')) >= 3 + } + + self.special_digits = pynini.union( + *[pynini.cross(k, v["alt"]) for k, v in special.items() if k in ["1", "4", "5"]] + ) + self.linh_digits = pynini.union(*[pynini.cross(k, special[k]["std"]) for k in ["1", "4", "5"]], self.digit) + + self.single_digit = self.digit + + self.two_digit = pynini.union( + self.teen, + self.ties + pynutil.delete("0"), + self.ties + + insert_space + + pynini.union(self.special_digits, pynini.union("2", "3", "6", "7", "8", "9") @ self.digit), + ) + + self.hundreds_pattern = pynini.union( + self.single_digit + insert_space + pynutil.insert(self.magnitudes["hundred"]) + pynutil.delete("00"), + self.single_digit + + insert_space + + pynutil.insert(self.magnitudes["hundred"]) + + pynutil.delete("0") + + insert_space + + pynutil.insert(self.magnitudes["linh"]) + + insert_space + + self.linh_digits, + self.single_digit + + insert_space + + pynutil.insert(self.magnitudes["hundred"]) + + insert_space + + self.two_digit, + ) + + self.hundreds = pynini.closure(NEMO_DIGIT, 3, 3) @ self.hundreds_pattern + + self.thousand = self._build_magnitude_pattern("thousand", 4, 6, 3) + self.million = self._build_magnitude_pattern("million", 7, 9, 6, self.thousand) + self.billion = self._build_magnitude_pattern("billion", 10, 12, 9, self.million) + + self.graph = pynini.union( + self.billion, self.million, self.thousand, self.hundreds, self.two_digit, self.single_digit, self.zero + ).optimize() + + self.single_digits_graph = self.single_digit | self.zero + self.graph_with_and = self.graph + + self.fst = self.add_tokens( + pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + pynutil.insert("integer: \"") + + self.graph + + pynutil.insert("\"") + ).optimize() + + def _build_magnitude_pattern(self, name, min_digits, max_digits, zero_count, prev_pattern=None): + magnitude_word = self.magnitudes[name] + + patterns = [] + for digits in range(min_digits, max_digits + 1): + leading_digits = digits - zero_count + leading_fst = {1: self.single_digit, 2: self.two_digit, 3: self.hundreds_pattern}.get( + leading_digits, self.hundreds_pattern + ) + + prefix = leading_fst + insert_space + pynutil.insert(magnitude_word) + + digit_patterns = [prefix + pynutil.delete("0" * zero_count)] + + if prev_pattern: + digit_patterns.append(prefix + insert_space + prev_pattern) + + trailing_patterns = [] + for trailing_zeros in range(zero_count): + remaining_digits = zero_count - trailing_zeros + if remaining_digits == 1: + trailing_patterns.append( + prefix + + pynutil.delete("0" * trailing_zeros) + + insert_space + + pynutil.insert(self.magnitudes["linh"]) + + insert_space + + self.linh_digits + ) + elif remaining_digits == 2: + trailing_patterns.append( + prefix + pynutil.delete("0" * trailing_zeros) + insert_space + self.two_digit + ) + elif remaining_digits == 3: + trailing_patterns.append( + prefix + pynutil.delete("0" * trailing_zeros) + insert_space + self.hundreds_pattern + ) + digit_patterns.extend(trailing_patterns) + + if name == "million" and digits == 7: + digit_patterns.extend( + [ + prefix + + pynutil.delete("00") + + insert_space + + self.single_digit + + insert_space + + pynutil.insert(self.magnitudes["thousand"]) + + pynutil.delete("00") + + insert_space + + pynutil.insert(self.magnitudes["linh"]) + + insert_space + + self.linh_digits, + prefix + + pynutil.delete("0") + + insert_space + + self.two_digit + + insert_space + + pynutil.insert(self.magnitudes["thousand"]) + + pynutil.delete("00") + + insert_space + + pynutil.insert(self.magnitudes["linh"]) + + insert_space + + self.linh_digits, + ] + ) + elif name == "billion" and digits == 10: + digit_patterns.append( + prefix + + pynutil.delete("00") + + insert_space + + self.single_digit + + insert_space + + pynutil.insert(self.magnitudes["million"]) + + pynutil.delete("00") + + insert_space + + self.single_digit + + insert_space + + pynutil.insert(self.magnitudes["thousand"]) + + insert_space + + self.hundreds_pattern + ) + + patterns.append(pynini.closure(NEMO_DIGIT, digits, digits) @ pynini.union(*digit_patterns)) + + return pynini.union(*patterns) diff --git a/nemo_text_processing/text_normalization/vi/taggers/punctuation.py b/nemo_text_processing/text_normalization/vi/taggers/punctuation.py new file mode 100644 index 000000000..1e08cb02d --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/punctuation.py @@ -0,0 +1,38 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation for Vietnamese + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + + # Common punctuation marks + # Use escape() for brackets since they are special regex chars + s = "!#$%&'()*+,-./:;<=>?@^_`{|}~–—――…»«„“›‹‚‘’⟨⟩" + punct = pynini.union(*s) + + # Create the punctuation transduction + graph = pynutil.insert('name: "') + punct + pynutil.insert('"') + + final_graph = pynutil.insert("punctuation { ") + graph + pynutil.insert(" }") + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/vi/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/vi/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..7c46c786a --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/tokenize_and_classify.py @@ -0,0 +1,94 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.vi.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.vi.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.vi.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.vi.taggers.word import WordFst +from nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join( + cache_dir, + f"vi_tn_{deterministic}_deterministic_{input_case}_tokenize.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating Vietnamese ClassifyFst grammars.") + + start_time = time.time() + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + logger.debug(f"cardinal: {time.time() - start_time: .2f}s -- {cardinal_graph.num_states()} nodes") + + start_time = time.time() + punctuation = PunctuationFst(deterministic=deterministic) + punct_graph = punctuation.fst + logger.debug(f"punct: {time.time() - start_time: .2f}s -- {punct_graph.num_states()} nodes") + + start_time = time.time() + whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic) + whitelist_graph = whitelist.fst + logger.debug(f"whitelist: {time.time() - start_time: .2f}s -- {whitelist_graph.num_states()} nodes") + + start_time = time.time() + word_graph = WordFst(deterministic=deterministic).fst + logger.debug(f"word: {time.time() - start_time: .2f}s -- {word_graph.num_states()} nodes") + + classify = ( + pynutil.add_weight(whitelist_graph, 0.8) + | pynutil.add_weight(cardinal_graph, 0.9) + | pynutil.add_weight(word_graph, 100) + ) + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure((delete_extra_space).ques + token_plus_punct) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/nemo_text_processing/text_normalization/vi/taggers/whitelist.py b/nemo_text_processing/text_normalization/vi/taggers/whitelist.py new file mode 100644 index 000000000..aed5e356a --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/whitelist.py @@ -0,0 +1,70 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space +from nemo_text_processing.text_normalization.vi.utils import get_abs_path, load_labels + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist for Vietnamese, e.g. + "h" -> tokens { name: "giờ" } + "p" -> tokens { name: "phút" } + "s" -> tokens { name: "giây" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file): + whitelist = load_labels(file) + if input_case == "lower_cased": + whitelist = [[x[0].lower()] + x[1:] for x in whitelist] + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) + if not deterministic and input_case != "lower_cased": + graph |= pynutil.add_weight( + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")), weight=0.0001 + ) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + # Add time units from time_units.tsv for better time handling + if not deterministic: + time_units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/time/time_units.tsv")) + graph |= time_units_graph + + self.graph = graph + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.final_graph + pynutil.insert("\"")).optimize() + + # Add tokens wrapper + self.fst = self.add_tokens(self.fst) diff --git a/nemo_text_processing/text_normalization/vi/taggers/word.py b/nemo_text_processing/text_normalization/vi/taggers/word.py new file mode 100644 index 000000000..f0be213c7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/word.py @@ -0,0 +1,34 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying Vietnamese words. + e.g. ngày -> name: "ngày" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/text_normalization/vi/utils.py b/nemo_text_processing/text_normalization/vi/utils.py new file mode 100644 index 000000000..332330921 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/utils.py @@ -0,0 +1,42 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/__init__.py b/nemo_text_processing/text_normalization/vi/verbalizers/__init__.py new file mode 100644 index 000000000..bc443be41 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/vi/verbalizers/cardinal.py new file mode 100644 index 000000000..530c3dfce --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/cardinal.py @@ -0,0 +1,55 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing Vietnamese cardinal numbers, e.g. + cardinal { negative: "true" integer: "hai mươi ba" } -> âm hai mươi ba + cardinal { integer: "một trăm" } -> một trăm + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + # Handle negative sign - Vietnamese uses "âm" for negative numbers + self.optional_sign = pynini.cross("negative: \"true\"", "âm ") + if not deterministic: + # Alternative ways to say negative in Vietnamese + self.optional_sign |= pynini.cross("negative: \"true\"", "trừ ") + self.optional_sign |= pynini.cross("negative: \"true\"", "âm ") + + self.optional_sign = pynini.closure(self.optional_sign + delete_space, 0, 1) + + # Handle the integer part + integer = pynini.closure(NEMO_NOT_QUOTE) + + self.integer = delete_space + pynutil.delete("\"") + integer + pynutil.delete("\"") + integer = pynutil.delete("integer:") + self.integer + + # Combine negative sign with integer + self.numbers = self.optional_sign + integer + + # Delete the token structure and create final FST + delete_tokens = self.delete_tokens(self.numbers) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/vi/verbalizers/verbalize.py new file mode 100644 index 000000000..fff63933e --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/verbalize.py @@ -0,0 +1,38 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.vi.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.vi.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + # Initialize verbalizers + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + whitelist = WhiteListFst(deterministic=deterministic) + whitelist_graph = whitelist.fst + + word = WordFst(deterministic=deterministic) + word_graph = word.fst + + # Combine all verbalizers + graph = cardinal_graph | whitelist_graph | word_graph + + self.fst = graph diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/vi/verbalizers/verbalize_final.py new file mode 100644 index 000000000..cd9ec39eb --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/verbalize_final.py @@ -0,0 +1,72 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.vi.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire Vietnamese sentence, e.g. + tokens { name: "xin" } tokens { cardinal { integer: "một trăm" } } tokens { name: "chào" } + -> xin một trăm chào + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"vi_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/whitelist.py b/nemo_text_processing/text_normalization/vi/verbalizers/whitelist.py new file mode 100644 index 000000000..6e0699827 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/whitelist.py @@ -0,0 +1,42 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist for Vietnamese + e.g. tokens { name: "giờ" } -> giờ + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/word.py b/nemo_text_processing/text_normalization/vi/verbalizers/word.py new file mode 100644 index 000000000..f9547acba --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/word.py @@ -0,0 +1,37 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing Vietnamese words. + e.g. tokens { name: "ngày" } -> ngày + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/tests/nemo_text_processing/vi/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..aad7ae8c1 --- /dev/null +++ b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,107 @@ +1~một +2~hai +3~ba +4~bốn +5~năm +6~sáu +7~bảy +8~tám +9~chín +10~mười +11~mười một +12~mười hai +15~mười lăm +18~mười tám +19~mười chín +20~hai mươi +21~hai mươi mốt +25~hai mươi lăm +30~ba mươi +34~ba mươi tư +44~bốn mươi tư +55~năm mươi lăm +67~sáu mươi bảy +70~bảy mươi +80~tám mươi +95~chín mươi lăm +100~một trăm +101~một trăm linh một +102~một trăm linh hai +104~một trăm linh bốn +105~một trăm linh năm +110~một trăm mười +111~một trăm mười một +120~một trăm hai mươi +123~một trăm hai mươi ba +200~hai trăm +201~hai trăm linh một +500~năm trăm +999~chín trăm chín mươi chín +1000~một nghìn +1001~một nghìn linh một +1020~một nghìn hai mươi +1095~một nghìn chín mươi lăm +1100~một nghìn một trăm +2000~hai nghìn +10000~mười nghìn +100000~một trăm nghìn +1000000~một triệu +2000000~hai triệu +1000000000~một tỷ +-1~âm một +-25~âm hai mươi lăm +-100~âm một trăm +-1000~âm một nghìn +0~không +1000~một nghìn +1001~một nghìn linh một +101~một trăm linh một +104~một trăm linh bốn +105~một trăm linh năm +24~hai mươi tư +35~ba mươi lăm +41~bốn mươi mốt +55~năm mươi lăm +91~chín mươi mốt +14~mười bốn +16~mười sáu +17~mười bảy +37~ba mươi bảy +47~bốn mươi bảy +57~năm mươi bảy +63~sáu mươi ba +79~bảy mươi chín +84~tám mươi tư +98~chín mươi tám +-123~âm một trăm hai mươi ba +-1001~âm một nghìn linh một +-104~âm một trăm linh bốn +1000001~một triệu linh một +1001001~một triệu một nghìn linh một +1050003~một triệu năm mươi nghìn linh ba +1000000001~một tỷ linh một +1001001101~một tỷ một triệu một nghìn một trăm linh một +300~ba trăm +400~bốn trăm +500~năm trăm +6000~sáu nghìn +7000~bảy nghìn +15000~mười lăm nghìn +300000~ba trăm nghìn +450000~bốn trăm năm mươi nghìn +5000000~năm triệu +700000000~bảy trăm triệu +31~ba mươi mốt +41~bốn mươi mốt +51~năm mươi mốt +61~sáu mươi mốt +71~bảy mươi mốt +81~tám mươi mốt +91~chín mươi mốt +5500000~năm triệu năm trăm nghìn +1000010~một triệu mười +1000100~một triệu một trăm +1000101~một triệu một trăm linh một +1010001~một triệu mười nghìn linh một +10000000000~mười tỷ +150~một trăm năm mươi \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/test_cardinal.py b/tests/nemo_text_processing/vi/test_cardinal.py index 0a888f84b..636932aed 100644 --- a/tests/nemo_text_processing/vi/test_cardinal.py +++ b/tests/nemo_text_processing/vi/test_cardinal.py @@ -12,32 +12,49 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pytest tests/nemo_text_processing/vi/test_cardinal.py --cpu --cache-clear import pytest from parameterized import parameterized -from ..utils import CACHE_DIR, parse_test_case_file +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -try: - from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer - - PYNINI_AVAILABLE = True -except (ImportError, ModuleNotFoundError): - PYNINI_AVAILABLE = False +from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestCardinal: - inverse_normalizer = ( - InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) if PYNINI_AVAILABLE else None - ) + inverse_normalizer = InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) @parameterized.expand(parse_test_case_file('vi/data_inverse_text_normalization/test_cases_cardinal.txt')) - @pytest.mark.skipif( - not PYNINI_AVAILABLE, - reason="`pynini` not installed, please install via nemo_text_processing/pynini_install.sh", - ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer( + input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + + normalizer_with_audio = ( + NormalizerWithAudio(input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) + if CACHE_DIR and RUN_AUDIO_BASED_TESTS + else None + ) + + @parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input}" + + if self.normalizer_with_audio: + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, + n_tagged=30, + punct_post_process=False, + ) + assert expected in pred_non_deterministic, f"input: {test_input}" diff --git a/tests/nemo_text_processing/vi/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/vi/test_sparrowhawk_inverse_text_normalization.sh index 751351cd4..684eb3b22 100644 --- a/tests/nemo_text_processing/vi/test_sparrowhawk_inverse_text_normalization.sh +++ b/tests/nemo_text_processing/vi/test_sparrowhawk_inverse_text_normalization.sh @@ -1,7 +1,7 @@ #! /bin/sh GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} -PROJECT_DIR=${2:-"/workspace/tests/en"} +PROJECT_DIR=${2:-"/workspace/tests"} runtest () { input=$1 diff --git a/tests/nemo_text_processing/vi/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/vi/test_sparrowhawk_normalization.sh new file mode 100644 index 000000000..d230b4642 --- /dev/null +++ b/tests/nemo_text_processing/vi/test_sparrowhawk_normalization.sh @@ -0,0 +1,77 @@ + +#! /bin/sh + +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests"} + +runtest () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + # read test file + while read testcase; do + IFS='~' read written spoken <<< $testcase + norm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + # trim white space + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + norm_pred="$(echo -e "${norm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$written" "$spoken" "$norm_pred" + done < "$input" +} + +testTNCardinal() { + input=$PROJECT_DIR/vi/data_text_normalization/test_cases_cardinal.txt + runtest $input +} + +# testTNDate() { +# input=$PROJECT_DIR/vi/data_text_normalization/test_cases_date.txt +# runtest $input +# } + +# testTNDecimal() { +# input=$PROJECT_DIR/vi/data_text_normalization/test_cases_decimal.txt +# runtest $input +# } + +# testTNOrdinal() { +# input=$PROJECT_DIR/vi/data_text_normalization/test_cases_ordinal.txt +# runtest $input +# } + +# testTNFraction() { +# input=$PROJECT_DIR/vi/data_text_normalization/test_cases_fraction.txt +# runtest $input +# } + +# testTNTime() { +# input=$PROJECT_DIR/vi/data_text_normalization/test_cases_time.txt +# runtest $input +# } + +# testTNMeasure() { +# input=$PROJECT_DIR/vi/data_text_normalization/test_cases_measure.txt +# runtest $input +# } + +# testTNMoney() { +# input=$PROJECT_DIR/vi/data_text_normalization/test_cases_money.txt +# runtest $input +# } + +# testTNTelephone() { +# input=$PROJECT_DIR/vi/data_text_normalization/test_cases_telephone.txt +# runtest $input +# } + +# testTNElectronic() { +# input=$PROJECT_DIR/vi/data_text_normalization/test_cases_electronic.txt +# runtest $input +# } + +# Load shUnit2 +. /workspace/shunit2/shunit2 diff --git a/tools/text_processing_deployment/pynini_export.py b/tools/text_processing_deployment/pynini_export.py index 6b82dfbec..bc19f428d 100644 --- a/tools/text_processing_deployment/pynini_export.py +++ b/tools/text_processing_deployment/pynini_export.py @@ -137,7 +137,7 @@ def parse_args(): if __name__ == '__main__': args = parse_args() - if args.language in ['pt', 'ru', 'vi', 'es_en', 'mr'] and args.grammars == 'tn_grammars': + if args.language in ['pt', 'ru', 'es_en', 'mr'] and args.grammars == 'tn_grammars': raise ValueError('Only ITN grammars could be deployed in Sparrowhawk for the selected languages.') TNPostProcessingFst = None ITNPostProcessingFst = None @@ -240,6 +240,10 @@ def parse_args(): from nemo_text_processing.inverse_text_normalization.vi.verbalizers.verbalize import ( VerbalizeFst as ITNVerbalizeFst, ) + from nemo_text_processing.text_normalization.vi.taggers.tokenize_and_classify import ( + ClassifyFst as TNClassifyFst, + ) + from nemo_text_processing.text_normalization.vi.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst elif args.language == 'zh': from nemo_text_processing.inverse_text_normalization.zh.taggers.tokenize_and_classify import ( ClassifyFst as ITNClassifyFst,