From 0db56356931f2e2341fef1a267762d61bea1f4a6 Mon Sep 17 00:00:00 2001 From: tjadamlee Date: Sun, 22 Dec 2019 10:57:44 +0800 Subject: [PATCH] first commit --- .clang-format | 1 + .flake8 | 101 + .gitignore | 8 + .pylintrc | 572 ++ .style.yapf | 2 + .vscode/settings.json | 3 + CONTRIBUTING.md | 32 + LICENSE | 433 ++ README.md | 164 + athena/__init__.py | 72 + athena/cmvn_main.py | 41 + athena/data/__init__.py | 22 + athena/data/datasets/__init__.py | 16 + athena/data/datasets/base.py | 146 + athena/data/datasets/language_set.py | 119 + athena/data/datasets/speech_recognition.py | 268 + .../data/datasets/speech_recognition_test.py | 47 + athena/data/datasets/speech_set.py | 208 + athena/data/feature_normalizer.py | 116 + athena/data/text_featurizer.py | 201 + athena/decode_main.py | 53 + athena/horovod_main.py | 38 + athena/layers/__init__.py | 16 + athena/layers/attention.py | 341 ++ athena/layers/commons.py | 113 + athena/layers/functional.py | 109 + athena/layers/transformer.py | 379 ++ athena/loss.py | 99 + athena/main.py | 170 + athena/metrics.py | 109 + athena/models/__init__.py | 16 + athena/models/base.py | 74 + athena/models/customized.py | 67 + athena/models/deep_speech.py | 91 + athena/models/masked_pc.py | 209 + athena/models/mtl_seq2seq.py | 143 + athena/models/rnn_lm.py | 87 + athena/models/speech_transformer.py | 319 ++ athena/solver.py | 238 + athena/tools/__init__.py | 16 + athena/tools/beam_search.py | 282 + athena/tools/ctc_scorer.py | 168 + athena/tools/lm_scorer.py | 79 + athena/transform/README.md | 227 + athena/transform/__init__.py | 20 + athena/transform/audio_featurizer.py | 93 + athena/transform/feats/__init__.py | 24 + athena/transform/feats/base_frontend.py | 58 + athena/transform/feats/cmvn.py | 86 + athena/transform/feats/cmvn_test.py | 52 + athena/transform/feats/fbank.py | 157 + athena/transform/feats/fbank_pitch.py | 215 + athena/transform/feats/fbank_pitch_test.py | 56 + athena/transform/feats/fbank_test.py | 102 + athena/transform/feats/framepow.py | 87 + athena/transform/feats/framepow_test.py | 74 + athena/transform/feats/mfcc.py | 153 + athena/transform/feats/mfcc_test.py | 76 + athena/transform/feats/ops/Makefile | 95 + athena/transform/feats/ops/__init__.py | 15 + .../feats/ops/kernels/complex_defines.h | 250 + .../feats/ops/kernels/delta_delta.cc | 121 + .../transform/feats/ops/kernels/delta_delta.h | 75 + .../feats/ops/kernels/delta_delta_op.cc | 84 + .../feats/ops/kernels/delta_delta_op_test.py | 302 + athena/transform/feats/ops/kernels/fbank.cc | 70 + athena/transform/feats/ops/kernels/fbank.h | 72 + .../transform/feats/ops/kernels/fbank_op.cc | 116 + .../feats/ops/kernels/fbank_op_test.py | 72 + .../transform/feats/ops/kernels/framepow.cc | 115 + athena/transform/feats/ops/kernels/framepow.h | 64 + .../feats/ops/kernels/framepow_op.cc | 88 + .../transform/feats/ops/kernels/mfcc_dct.cc | 106 + athena/transform/feats/ops/kernels/mfcc_dct.h | 53 + .../feats/ops/kernels/mfcc_dct_op.cc | 119 + .../feats/ops/kernels/mfcc_mel_filterbank.cc | 211 + .../feats/ops/kernels/mfcc_mel_filterbank.h | 65 + athena/transform/feats/ops/kernels/pitch.cc | 1068 ++++ athena/transform/feats/ops/kernels/pitch.h | 187 + .../transform/feats/ops/kernels/pitch_op.cc | 152 + .../transform/feats/ops/kernels/resample.cc | 313 ++ athena/transform/feats/ops/kernels/resample.h | 123 + .../transform/feats/ops/kernels/spectrum.cc | 210 + athena/transform/feats/ops/kernels/spectrum.h | 94 + .../feats/ops/kernels/spectrum_op.cc | 113 + .../feats/ops/kernels/spectrum_op_test.py | 68 + .../transform/feats/ops/kernels/speed_op.cc | 84 + .../feats/ops/kernels/support_functions.cc | 725 +++ .../feats/ops/kernels/support_functions.h | 172 + athena/transform/feats/ops/kernels/x_ops.cc | 292 + athena/transform/feats/ops/py_x_ops.py | 34 + athena/transform/feats/pitch.py | 176 + athena/transform/feats/pitch_test.py | 92 + athena/transform/feats/read_wav.py | 84 + athena/transform/feats/read_wav_test.py | 57 + athena/transform/feats/spectrum.py | 133 + athena/transform/feats/spectrum_test.py | 87 + athena/transform/feats/write_wav.py | 67 + athena/transform/feats/write_wav_test.py | 55 + athena/utils/__init__.py | 16 + athena/utils/checkpoint.py | 76 + athena/utils/data_queue.py | 104 + athena/utils/hparam.py | 695 +++ athena/utils/hparam_test.py | 521 ++ athena/utils/learning_rate.py | 131 + athena/utils/metric_check.py | 104 + athena/utils/misc.py | 167 + athena/utils/vocabs/ch-en.vocab | 5005 +++++++++++++++++ athena/utils/vocabs/ch.vocab | 4977 ++++++++++++++++ athena/utils/vocabs/en.vocab | 30 + docs/README.md | 158 + docs/TheTrainningEfficiency.md | 31 + docs/development/contributing.md | 53 + docs/transform/speech_feature.md | 78 + docs/transform/user_manual.md | 166 + examples/asr/README.md | 22 + examples/asr/aishell/data/vocab | 4232 ++++++++++++++ examples/asr/aishell/local/prepare_data.py | 112 + examples/asr/aishell/mtl_transformer.json | 70 + examples/asr/aishell/mtl_transformer_sp.json | 71 + examples/asr/hkust/README.md | 18 + examples/asr/hkust/data/vocab | 3650 ++++++++++++ examples/asr/hkust/deep_speech.json | 47 + examples/asr/hkust/local/prepare_data.py | 203 + examples/asr/hkust/local/segment_word.py | 52 + examples/asr/hkust/mpc.json | 47 + examples/asr/hkust/mtl_transformer.json | 70 + examples/asr/hkust/mtl_transformer_sp.json | 71 + examples/asr/hkust/rnnlm.json | 49 + examples/asr/hkust/run.sh | 71 + examples/asr/hkust/transformer.json | 55 + .../data/librispeech_unigram5000.model | Bin 0 -> 325118 bytes examples/asr/librispeech/prepare_data.py | 171 + examples/asr/librispeech/transformer.json | 49 + .../asr/switchboard_fisher/prepare_data.py | 354 ++ .../translate/spa-eng-example/prepare_data.py | 46 + requirements.txt | 12 + setup.py | 67 + tools/env.sh | 20 + tools/install.sh | 4 + tools/install_kenlm.sh | 8 + tools/install_sph2pipe.sh | 10 + 142 files changed, 35640 insertions(+) create mode 100644 .clang-format create mode 100644 .flake8 create mode 100644 .gitignore create mode 100644 .pylintrc create mode 100644 .style.yapf create mode 100644 .vscode/settings.json create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 athena/__init__.py create mode 100644 athena/cmvn_main.py create mode 100644 athena/data/__init__.py create mode 100644 athena/data/datasets/__init__.py create mode 100644 athena/data/datasets/base.py create mode 100644 athena/data/datasets/language_set.py create mode 100644 athena/data/datasets/speech_recognition.py create mode 100644 athena/data/datasets/speech_recognition_test.py create mode 100644 athena/data/datasets/speech_set.py create mode 100644 athena/data/feature_normalizer.py create mode 100644 athena/data/text_featurizer.py create mode 100644 athena/decode_main.py create mode 100644 athena/horovod_main.py create mode 100644 athena/layers/__init__.py create mode 100644 athena/layers/attention.py create mode 100644 athena/layers/commons.py create mode 100644 athena/layers/functional.py create mode 100644 athena/layers/transformer.py create mode 100644 athena/loss.py create mode 100644 athena/main.py create mode 100644 athena/metrics.py create mode 100644 athena/models/__init__.py create mode 100644 athena/models/base.py create mode 100644 athena/models/customized.py create mode 100644 athena/models/deep_speech.py create mode 100644 athena/models/masked_pc.py create mode 100644 athena/models/mtl_seq2seq.py create mode 100644 athena/models/rnn_lm.py create mode 100644 athena/models/speech_transformer.py create mode 100644 athena/solver.py create mode 100644 athena/tools/__init__.py create mode 100644 athena/tools/beam_search.py create mode 100644 athena/tools/ctc_scorer.py create mode 100644 athena/tools/lm_scorer.py create mode 100644 athena/transform/README.md create mode 100644 athena/transform/__init__.py create mode 100644 athena/transform/audio_featurizer.py create mode 100644 athena/transform/feats/__init__.py create mode 100644 athena/transform/feats/base_frontend.py create mode 100644 athena/transform/feats/cmvn.py create mode 100644 athena/transform/feats/cmvn_test.py create mode 100644 athena/transform/feats/fbank.py create mode 100644 athena/transform/feats/fbank_pitch.py create mode 100644 athena/transform/feats/fbank_pitch_test.py create mode 100644 athena/transform/feats/fbank_test.py create mode 100644 athena/transform/feats/framepow.py create mode 100644 athena/transform/feats/framepow_test.py create mode 100644 athena/transform/feats/mfcc.py create mode 100644 athena/transform/feats/mfcc_test.py create mode 100644 athena/transform/feats/ops/Makefile create mode 100644 athena/transform/feats/ops/__init__.py create mode 100644 athena/transform/feats/ops/kernels/complex_defines.h create mode 100644 athena/transform/feats/ops/kernels/delta_delta.cc create mode 100644 athena/transform/feats/ops/kernels/delta_delta.h create mode 100644 athena/transform/feats/ops/kernels/delta_delta_op.cc create mode 100644 athena/transform/feats/ops/kernels/delta_delta_op_test.py create mode 100644 athena/transform/feats/ops/kernels/fbank.cc create mode 100644 athena/transform/feats/ops/kernels/fbank.h create mode 100644 athena/transform/feats/ops/kernels/fbank_op.cc create mode 100644 athena/transform/feats/ops/kernels/fbank_op_test.py create mode 100644 athena/transform/feats/ops/kernels/framepow.cc create mode 100644 athena/transform/feats/ops/kernels/framepow.h create mode 100644 athena/transform/feats/ops/kernels/framepow_op.cc create mode 100644 athena/transform/feats/ops/kernels/mfcc_dct.cc create mode 100644 athena/transform/feats/ops/kernels/mfcc_dct.h create mode 100644 athena/transform/feats/ops/kernels/mfcc_dct_op.cc create mode 100644 athena/transform/feats/ops/kernels/mfcc_mel_filterbank.cc create mode 100644 athena/transform/feats/ops/kernels/mfcc_mel_filterbank.h create mode 100644 athena/transform/feats/ops/kernels/pitch.cc create mode 100644 athena/transform/feats/ops/kernels/pitch.h create mode 100644 athena/transform/feats/ops/kernels/pitch_op.cc create mode 100644 athena/transform/feats/ops/kernels/resample.cc create mode 100644 athena/transform/feats/ops/kernels/resample.h create mode 100644 athena/transform/feats/ops/kernels/spectrum.cc create mode 100644 athena/transform/feats/ops/kernels/spectrum.h create mode 100644 athena/transform/feats/ops/kernels/spectrum_op.cc create mode 100644 athena/transform/feats/ops/kernels/spectrum_op_test.py create mode 100644 athena/transform/feats/ops/kernels/speed_op.cc create mode 100644 athena/transform/feats/ops/kernels/support_functions.cc create mode 100644 athena/transform/feats/ops/kernels/support_functions.h create mode 100644 athena/transform/feats/ops/kernels/x_ops.cc create mode 100644 athena/transform/feats/ops/py_x_ops.py create mode 100644 athena/transform/feats/pitch.py create mode 100644 athena/transform/feats/pitch_test.py create mode 100644 athena/transform/feats/read_wav.py create mode 100644 athena/transform/feats/read_wav_test.py create mode 100644 athena/transform/feats/spectrum.py create mode 100644 athena/transform/feats/spectrum_test.py create mode 100644 athena/transform/feats/write_wav.py create mode 100644 athena/transform/feats/write_wav_test.py create mode 100644 athena/utils/__init__.py create mode 100644 athena/utils/checkpoint.py create mode 100644 athena/utils/data_queue.py create mode 100644 athena/utils/hparam.py create mode 100644 athena/utils/hparam_test.py create mode 100644 athena/utils/learning_rate.py create mode 100644 athena/utils/metric_check.py create mode 100644 athena/utils/misc.py create mode 100644 athena/utils/vocabs/ch-en.vocab create mode 100644 athena/utils/vocabs/ch.vocab create mode 100644 athena/utils/vocabs/en.vocab create mode 100644 docs/README.md create mode 100644 docs/TheTrainningEfficiency.md create mode 100644 docs/development/contributing.md create mode 100644 docs/transform/speech_feature.md create mode 100644 docs/transform/user_manual.md create mode 100644 examples/asr/README.md create mode 100644 examples/asr/aishell/data/vocab create mode 100644 examples/asr/aishell/local/prepare_data.py create mode 100644 examples/asr/aishell/mtl_transformer.json create mode 100644 examples/asr/aishell/mtl_transformer_sp.json create mode 100644 examples/asr/hkust/README.md create mode 100644 examples/asr/hkust/data/vocab create mode 100644 examples/asr/hkust/deep_speech.json create mode 100644 examples/asr/hkust/local/prepare_data.py create mode 100644 examples/asr/hkust/local/segment_word.py create mode 100644 examples/asr/hkust/mpc.json create mode 100644 examples/asr/hkust/mtl_transformer.json create mode 100644 examples/asr/hkust/mtl_transformer_sp.json create mode 100644 examples/asr/hkust/rnnlm.json create mode 100644 examples/asr/hkust/run.sh create mode 100644 examples/asr/hkust/transformer.json create mode 100644 examples/asr/librispeech/data/librispeech_unigram5000.model create mode 100644 examples/asr/librispeech/prepare_data.py create mode 100644 examples/asr/librispeech/transformer.json create mode 100644 examples/asr/switchboard_fisher/prepare_data.py create mode 100644 examples/translate/spa-eng-example/prepare_data.py create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 tools/env.sh create mode 100644 tools/install.sh create mode 100644 tools/install_kenlm.sh create mode 100644 tools/install_sph2pipe.sh diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..f6cb8ad9 --- /dev/null +++ b/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: Google diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..14159d95 --- /dev/null +++ b/.flake8 @@ -0,0 +1,101 @@ +# .flake8 +# +# DESCRIPTION +# Configuration file for the python linter flake8. +# +# This configuration is based on the generic +# configuration published on GitHub. +# +# AUTHOR krnd +# VERSION v1.0 +# +# SEE ALSO +# http://flake8.pycqa.org/en/latest/user/options.html +# http://flake8.pycqa.org/en/latest/user/error-codes.html +# http://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes +# http://gist.github.com/krnd +# + +[flake8] + +# Specify the number of subprocesses that Flake8 will use to run checks in parallel. +jobs = auto + +# Increase the verbosity of Flake8’s output. +verbose = 0 + +# Decrease the verbosity of Flake8’s output. +quiet = 0 + +# Select the formatter used to display errors to the user. +format = default + +# Print the total number of errors. +count = True + +# Print the source code generating the error/warning in question. +show-source = True + +# Count the number of occurrences of each error/warning code and print a report. +statistics = True + + +# Redirect all output to the specified file. +output-file = /tmp/flake8.log + +# Also print output to stdout if output-file has been configured. +tee = True + +# Provide a comma-separated list of glob patterns to exclude from checks. +exclude = + # git folder + .git, + # python cache + __pycache__, + tools + +# Provide a comma-separate list of glob patterns to include for checks. +filename = *.py + +# Provide a custom list of builtin functions, objects, names, etc. +builtins = + +# Report all errors, even if it is on the same line as a `# NOQA` comment. +disable-noqa = False + +# Set the maximum length that any line (with some exceptions) may be. +max-line-length = 100 + +# Set the maximum allowed McCabe complexity value for a block of code. +max-complexity = 10 + +# Toggle whether pycodestyle should enforce matching the indentation of the opening bracket’s line. +# incluences E131 and E133 +hang-closing = True + + +# ERROR CODES +# +# E/W - PEP8 errors/warnings (pycodestyle) +# F - linting errors (pyflakes) +# C - McCabe complexity error (mccabe) +# +# W503 - line break before binary operator + +# Specify a list of codes to ignore. +ignore = W503 + +# Specify the list of error codes you wish Flake8 to report. +select = E9,F6,F81,F82,F83,F9 + +# Enable off-by-default extensions. +enable-extensions = + +# Enable PyFlakes syntax checking of doctests in docstrings. +doctests = True + +# Specify which files are checked by PyFlakes for doctest syntax. +include-in-doctest = + +# Specify which files are not to be checked by PyFlakes for doctest syntax. +exclude-in-doctest = diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..15307afb --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.idea +__pycache__ +*.pyc +tags +.vscode + +/tools/sph2pipe_v2.5.tar.gz +/tools/sph2pipe_v2.5 diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..31cf6b00 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,572 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + non-ascii-bytes-literal, + raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + deprecated-itertools-function, + deprecated-types-field, + next-method-defined, + dict-items-not-iterating, + dict-keys-not-iterating, + dict-values-not-iterating, + deprecated-operator-function, + deprecated-urllib-function, + xreadlines-attribute, + deprecated-sys-function, + exception-escape, + comprehension-escape, + bad-indentation, + bad-continuation + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package.. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[LOGGING] + +# Format style used to check logging format string. `old` means using % +# formatting, while `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[STRING] + +# This flag controls whether the implicit-str-concat-in-sequence should +# generate a warning on implicit string concatenation in sequences defined over +# several lines. +check-str-concat-over-line-jumps=no + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma, + dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=20 + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement. +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 00000000..b206b75b --- /dev/null +++ b/.style.yapf @@ -0,0 +1,2 @@ +[style] +based_on_style = chromium diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..500bc700 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.linting.pylintEnabled": true +} \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..6b0ac6b5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,32 @@ +# Contribution Guideline + +Thanks for considering to contribute this project. All issues and pull requests are highly appreciated. + +## Pull Requests + +Before sending pull request to this project, please read and follow guidelines below. + +1. Branch: We accept pull request on `master` branch. +2. Coding style: Follow the coding style used in VirtualAPK. +3. Commit message: Use English and be aware of your spell. +4. Test: Make sure to test your code. + +Add device mode, API version, related log, screenshots and other related information in your pull request if possible. + +NOTE: We assume all your contribution can be licensed under the [Apache License 2.0](https://github.com/didichuxing/athena/tree/master/LICENSE). + +## Issues + +We love clearly described issues. :) + +Following information can help us to resolve the issue faster. + +* Device mode and hardware information. +* API version. +* Logs. +* Screenshots. +* Steps to reproduce the issue. + +## Coding Styles + +Please follow the coding styles [here](https://git.xiaojukeji.com/speech-am/athena/tree/master/docs/development/contributing.md) diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..5e8d4011 --- /dev/null +++ b/LICENSE @@ -0,0 +1,433 @@ + Apache License + + Version 2.0, January 2004 + + http://www.apache.org/licenses/ + + + + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + + + +1. Definitions. + + + + + "License" shall mean the terms and conditions for use, reproduction, + + and distribution as defined by Sections 1 through 9 of this document. + + + + + "Licensor" shall mean the copyright owner or entity authorized by + + the copyright owner that is granting the License. + + + + + "Legal Entity" shall mean the union of the acting entity and all + + other entities that control, are controlled by, or are under common + + control with that entity. For the purposes of this definition, + + "control" means (i) the power, direct or indirect, to cause the + + direction or management of such entity, whether by contract or + + otherwise, or (ii) ownership of fifty percent (50%) or more of the + + outstanding shares, or (iii) beneficial ownership of such entity. + + + + + "You" (or "Your") shall mean an individual or Legal Entity + + exercising permissions granted by this License. + + + + + "Source" form shall mean the preferred form for making modifications, + + including but not limited to software source code, documentation + + source, and configuration files. + + + + + "Object" form shall mean any form resulting from mechanical + + transformation or translation of a Source form, including but + + not limited to compiled object code, generated documentation, + + and conversions to other media types. + + + + + "Work" shall mean the work of authorship, whether in Source or + + Object form, made available under the License, as indicated by a + + copyright notice that is included in or attached to the work + + (an example is provided in the Appendix below). + + + + + "Derivative Works" shall mean any work, whether in Source or Object + + form, that is based on (or derived from) the Work and for which the + + editorial revisions, annotations, elaborations, or other modifications + + represent, as a whole, an original work of authorship. For the purposes + + of this License, Derivative Works shall not include works that remain + + separable from, or merely link (or bind by name) to the interfaces of, + + the Work and Derivative Works thereof. + + + + + "Contribution" shall mean any work of authorship, including + + the original version of the Work and any modifications or additions + + to that Work or Derivative Works thereof, that is intentionally + + submitted to Licensor for inclusion in the Work by the copyright owner + + or by an individual or Legal Entity authorized to submit on behalf of + + the copyright owner. For the purposes of this definition, "submitted" + + means any form of electronic, verbal, or written communication sent + + to the Licensor or its representatives, including but not limited to + + communication on electronic mailing lists, source code control systems, + + and issue tracking systems that are managed by, or on behalf of, the + + Licensor for the purpose of discussing and improving the Work, but + + excluding communication that is conspicuously marked or otherwise + + designated in writing by the copyright owner as "Not a Contribution." + + + + + "Contributor" shall mean Licensor and any individual or Legal Entity + + on behalf of whom a Contribution has been received by Licensor and + + subsequently incorporated within the Work. + + + + +2. Grant of Copyright License. Subject to the terms and conditions of + + this License, each Contributor hereby grants to You a perpetual, + + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + + copyright license to reproduce, prepare Derivative Works of, + + publicly display, publicly perform, sublicense, and distribute the + + Work and such Derivative Works in Source or Object form. + + + + +3. Grant of Patent License. Subject to the terms and conditions of + + this License, each Contributor hereby grants to You a perpetual, + + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + + (except as stated in this section) patent license to make, have made, + + use, offer to sell, sell, import, and otherwise transfer the Work, + + where such license applies only to those patent claims licensable + + by such Contributor that are necessarily infringed by their + + Contribution(s) alone or by combination of their Contribution(s) + + with the Work to which such Contribution(s) was submitted. If You + + institute patent litigation against any entity (including a + + cross-claim or counterclaim in a lawsuit) alleging that the Work + + or a Contribution incorporated within the Work constitutes direct + + or contributory patent infringement, then any patent licenses + + granted to You under this License for that Work shall terminate + + as of the date such litigation is filed. + + + + +4. Redistribution. You may reproduce and distribute copies of the + + Work or Derivative Works thereof in any medium, with or without + + modifications, and in Source or Object form, provided that You + + meet the following conditions: + + + + + (a) You must give any other recipients of the Work or + + Derivative Works a copy of this License; and + + + + + (b) You must cause any modified files to carry prominent notices + + stating that You changed the files; and + + + + + (c) You must retain, in the Source form of any Derivative Works + + that You distribute, all copyright, patent, trademark, and + + attribution notices from the Source form of the Work, + + excluding those notices that do not pertain to any part of + + the Derivative Works; and + + + + + (d) If the Work includes a "NOTICE" text file as part of its + + distribution, then any Derivative Works that You distribute must + + include a readable copy of the attribution notices contained + + within such NOTICE file, excluding those notices that do not + + pertain to any part of the Derivative Works, in at least one + + of the following places: within a NOTICE text file distributed + + as part of the Derivative Works; within the Source form or + + documentation, if provided along with the Derivative Works; or, + + within a display generated by the Derivative Works, if and + + wherever such third-party notices normally appear. The contents + + of the NOTICE file are for informational purposes only and + + do not modify the License. You may add Your own attribution + + notices within Derivative Works that You distribute, alongside + + or as an addendum to the NOTICE text from the Work, provided + + that such additional attribution notices cannot be construed + + as modifying the License. + + + + + You may add Your own copyright statement to Your modifications and + + may provide additional or different license terms and conditions + + for use, reproduction, or distribution of Your modifications, or + + for any such Derivative Works as a whole, provided Your use, + + reproduction, and distribution of the Work otherwise complies with + + the conditions stated in this License. + + + + +5. Submission of Contributions. Unless You explicitly state otherwise, + + any Contribution intentionally submitted for inclusion in the Work + + by You to the Licensor shall be under the terms and conditions of + + this License, without any additional terms or conditions. + + Notwithstanding the above, nothing herein shall supersede or modify + + the terms of any separate license agreement you may have executed + + with Licensor regarding such Contributions. + + + + +6. Trademarks. This License does not grant permission to use the trade + + names, trademarks, service marks, or product names of the Licensor, + + except as required for reasonable and customary use in describing the + + origin of the Work and reproducing the content of the NOTICE file. + + + + +7. Disclaimer of Warranty. Unless required by applicable law or + + agreed to in writing, Licensor provides the Work (and each + + Contributor provides its Contributions) on an "AS IS" BASIS, + + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + + implied, including, without limitation, any warranties or conditions + + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + + PARTICULAR PURPOSE. You are solely responsible for determining the + + appropriateness of using or redistributing the Work and assume any + + risks associated with Your exercise of permissions under this License. + + + + +8. Limitation of Liability. In no event and under no legal theory, + + whether in tort (including negligence), contract, or otherwise, + + unless required by applicable law (such as deliberate and grossly + + negligent acts) or agreed to in writing, shall any Contributor be + + liable to You for damages, including any direct, indirect, special, + + incidental, or consequential damages of any character arising as a + + result of this License or out of the use or inability to use the + + Work (including but not limited to damages for loss of goodwill, + + work stoppage, computer failure or malfunction, or any and all + + other commercial damages or losses), even if such Contributor + + has been advised of the possibility of such damages. + + + + +9. Accepting Warranty or Additional Liability. While redistributing + + the Work or Derivative Works thereof, You may choose to offer, + + and charge a fee for, acceptance of support, warranty, indemnity, + + or other liability obligations and/or rights consistent with this + + License. However, in accepting such obligations, You may act only + + on Your own behalf and on Your sole responsibility, not on behalf + + of any other Contributor, and only if You agree to indemnify, + + defend, and hold each Contributor harmless for any liability + + incurred by, or claims asserted against, such Contributor by reason + + of your accepting any such warranty or additional liability. + + + + +END OF TERMS AND CONDITIONS + + + + +APPENDIX: How to apply the Apache License to your work. + + + + + To apply the Apache License to your work, attach the following + + boilerplate notice, with the fields enclosed by brackets "{}" + + replaced with your own identifying information. (Don't include + + the brackets!) The text should be enclosed in the appropriate + + comment syntax for the file format. We also recommend that a + + file or class name and description of purpose be included on the + + same "printed page" as the copyright notice for easier + + identification within third-party archives. + + + + +Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. All rights reserved. + + + + +Licensed under the Apache License, Version 2.0 (the "License"); + +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at + + + + + http://www.apache.org/licenses/LICENSE-2.0 + + + + +Unless required by applicable law or agreed to in writing, software + +distributed under the License is distributed on an "AS IS" BASIS, + +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and + +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 00000000..b40976e7 --- /dev/null +++ b/README.md @@ -0,0 +1,164 @@ + +# Athena + +*Athena* is an open-source implementation of end-to-end Automatic Speech Recognition (ASR) engine. Currently this project supports training and decoding of Connectionist Temporal Classification (CTC) based model, transformer-basesd encoder-decoder model and Hybrid CTC/attention based model, and MPC based unsupervised pretraning. + +Our vision is to empower both industrial application and academic research on end-to-end models for speech recognition. To make ASR accessible to everyone, we're also releasing some example implementation based on some opensource dataset, like HKSUT, Librispeech + +All of our models are implemented in Tensorflow>=2.0.0. + + +## Table of Contents + +- [Athena](#athena) + - [Table of Contents](#table-of-contents) + - [Key Features](#key-features) + - [Installation](#installation) + - [Data Preparation](#data-preparation) + - [Create Manifest](#create-manifest) + - [Training](#training) + - [Setting the Configuration File](#setting-the-configuration-file) + - [Train a Model](#train-a-model) + - [Results](#results) + - [Directory Structure](#directory-structure) + +## Key Features + +- Hybrid CTC/Transformer based end-to-end ASR +- Speech-Transformer +- MPC based unsupervised pretraining + +## Installation + +This project has only been tested on Python 3. We recommend creating a virtual environment and installing the python requirements there. + +```bash +git clone https://github.com/didi/athena.git +cd athena +pip install -r requirements.txt +python setup.py bdist_wheel sdist +python -m pip install --ignore-installed dist/athena-0.1.0*.whl +source ./tools/env.sh +``` + +Notes: + +- If you see errors such as `ERROR: Cannot uninstall 'wrapt'` while installing TensorFlow, try updating it using command `conda update wrapt`. Same for similar dependencies such as `entrypoints`, `llvmlite` and so on. +- You may want to make sure you have `g++` version 7 or above to make sure you can successfully install TensorFlow. + +## Data Preparation + +### Create Manifest + +Athena accepts a textual manifest file as data set interface, which describes speech data set in csv format. In such file, each line contains necessary meta data (e.g. key, audio path, transcription) of a speech audio. For custom data, such manifest file needs to be prepared first. An example is shown as follows: + +```csv +wav_filename wav_length_ms transcript +/dataset/train-clean-100-wav/374-180298-0000.wav 465004 chapter sixteen i might have told you of the beginning of this liaison in a few lines but i wanted you to see every step by which we came i to agree to whatever marguerite wished +/dataset/train-clean-100-wav/374-180298-0001.wav 514764 marguerite to be unable to live apart from me it was the day after the evening when she came to see me that i sent her manon lescaut from that time seeing that i could not change my mistress's life i changed my own +/dataset/train-clean-100-wav/374-180298-0002.wav 425484 i wished above all not to leave myself time to think over the position i had accepted for in spite of myself it was a great distress to me thus my life generally so calm +/dataset/train-clean-100-wav/374-180298-0003.wav 356044 assumed all at once an appearance of noise and disorder never believe however disinterested the love of a kept woman may be that it will cost one nothing +``` + +## Training + +### Setting the Configuration File + +All of our training/ inference configurations are written in config.json. Below is an example configuration file with comments to help you understand. + +```json +{ + "batch_size":32, + "num_epochs":20, + "sorta_epoch":1, + "ckpt":"examples/asr/hkust/ckpts/transformer", + + "solver_gpu":[0], + "solver_config":{ + "clip_norm":100, + "log_interval":10, + "enable_tf_function":true + }, + + "model":"speech_transformer", + "num_classes": null, + "pretrained_model": null, + "model_config":{ + "return_encoder_output":false, + "num_filters":512, + "d_model":512, + "num_heads":8, + "num_encoder_layers":12, + "num_decoder_layers":6, + "dff":1280, + "rate":0.1, + "label_smoothing_rate":0.0 + }, + + "optimizer":"warmup_adam", + "optimizer_config":{ + "d_model":512, + "warmup_steps":8000, + "k":0.5 + }, + + "dataset_builder": "speech_recognition_dataset", + "dataset_config":{ + "audio_config":{ + "type":"Fbank", + "filterbank_channel_count":40, + "local_cmvn":false + }, + "cmvn_file":"examples/asr/hkust/data/cmvn", + "vocab_file":"examples/asr/hkust/data/vocab", + "input_length_range":[10, 5000] + }, + "train_csv":"/tmp-data/dataset/opensource/hkust/train.csv", + "dev_csv":"/tmp-data/dataset/opensource/hkust/dev.csv", + "test_csv":"/tmp-data/dataset/opensource/hkust/dev.csv" +} +``` + +### Train a Model + +With all the above preparation done, training becomes straight-forward. `athena/main.py` is the entry point of the training module. Just run `python athena/main.py ` + +Please install Horovod and MPI at first, if you want to train model using multi-gpu. See the [Horovod page](https://github.com/horovod/horovod) for more instructions. + +To run on a machine with 4 GPUs with Athona: +`$ horovodrun -np 4 -H localhost:4 python athena/horovod_main.py ` + +To run on 4 machines with 4 GPUs each with Athena: +`$ horovodrun -np 16 -H server1:4,server2:4,server3:4,server4:4 python athena/horovod_main.py ` + +## Results + +Language | Model Name | Training Data | Hours of Speech | WER/% +:-----------: | :------------: | :----------: | -------: | -------: +English | Transformer | [LibriSpeech Dataset](http://www.openslr.org/12/) | 960 h | +Mandarin | Transformer | HKUST Dataset | 151 h | + +## Directory Structure + +Below is the basic directory structure for Athena + +```bash +|-- Athena +| |-- data # - root directory for input-related operations +| | |-- datasets # custom datasets for ASR and pretraining +| |-- layers # some layers +| |-- models # some models +| |-- tools # contains various tools, e.g. decoding tools +| |-- transform # custom featureizer based on C++ +| | |-- feats +| | | |-- ops # c++ code on tensorflow ops +| |-- utils # utils, e.g. checkpoit, learning_rate, metric, etc +|-- docs # docs +|-- examples # example scripts for ASR, TTS, etc +| |-- asr # each subdirectory contains a data preparation scripts and a run script for the task +| |-- aishell +| |-- hkust +| |-- librispeech +| |-- switchboard_fisher +|-- tools # need to source env.sh before training +``` diff --git a/athena/__init__.py b/athena/__init__.py new file mode 100644 index 00000000..45fc6b48 --- /dev/null +++ b/athena/__init__.py @@ -0,0 +1,72 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" module """ +# data +from .data import SpeechRecognitionDatasetBuilder +from .data import SpeechDatasetBuilder +from .data import LanguageDatasetBuilder +from .data import FeatureNormalizer +from .data.text_featurizer import TextFeaturizer + +# layers +from .layers.functional import make_positional_encoding +from .layers.functional import collapse4d +from .layers.functional import gelu +from .layers.commons import PositionalEncoding +from .layers.commons import Collapse4D +from .layers.commons import TdnnLayer +from .layers.commons import Gelu +from .layers.attention import MultiHeadAttention +from .layers.attention import BahdanauAttention +from .layers.attention import HanAttention +from .layers.attention import MatchAttention +from .layers.transformer import Transformer +from .layers.transformer import TransformerEncoder +from .layers.transformer import TransformerDecoder +from .layers.transformer import TransformerEncoderLayer +from .layers.transformer import TransformerDecoderLayer + +# models +from .models.base import BaseModel +from .models.speech_transformer import SpeechTransformer, SpeechTransformer2 +from .models.masked_pc import MaskedPredictCoding +from .models.deep_speech import DeepSpeechModel +from .models.mtl_seq2seq import MtlTransformerCtc +from .models.rnn_lm import RNNLM + +# solver & loss & accuracy +from .solver import BaseSolver +from .solver import HorovodSolver +from .solver import DecoderSolver +from .loss import CTCLoss +from .loss import Seq2SeqSparseCategoricalCrossentropy +from .metrics import CTCAccuracy +from .metrics import Seq2SeqSparseCategoricalAccuracy + +# utils +from .utils.checkpoint import Checkpoint +from .utils.learning_rate import WarmUpLearningSchedule, WarmUpAdam +from .utils.learning_rate import ( + ExponentialDecayLearningRateSchedule, + ExponentialDecayAdam, +) +from .utils.hparam import HParams, register_and_parse_hparams +from .utils.misc import generate_square_subsequent_mask +from .utils.misc import get_wave_file_length +from .utils.misc import set_default_summary_writer + +# tools +from .tools.beam_search import BeamSearchDecoder diff --git a/athena/cmvn_main.py b/athena/cmvn_main.py new file mode 100644 index 00000000..6dfe58d5 --- /dev/null +++ b/athena/cmvn_main.py @@ -0,0 +1,41 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support tensorflow 2.0 +# pylint: disable=invalid-name, no-member +r""" a sample implementation of LAS for HKUST """ +import sys +import json +import tensorflow as tf +from absl import logging +from athena.main import parse_config, SUPPORTED_DATASET_BUILDER + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + if len(sys.argv) < 3: + logging.warning('Usage: python {} config_json_file data_csv_file'.format(sys.argv[0])) + sys.exit() + tf.random.set_seed(1) + + jsonfile = sys.argv[1] + with open(jsonfile) as file: + config = json.load(file) + p = parse_config(config) + if "speed_permutation" in p.dataset_config: + p.dataset_config['speed_permutation'] = [1.0] + csv_file = sys.argv[2] + dataset_builder = SUPPORTED_DATASET_BUILDER[p.dataset_builder](p.dataset_config) + dataset_builder.load_csv(csv_file).compute_cmvn_if_necessary(True) diff --git a/athena/data/__init__.py b/athena/data/__init__.py new file mode 100644 index 00000000..baa80991 --- /dev/null +++ b/athena/data/__init__.py @@ -0,0 +1,22 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" data """ +from .datasets.speech_recognition import SpeechRecognitionDatasetBuilder +from .datasets.speech_set import SpeechDatasetBuilder +from .datasets.language_set import LanguageDatasetBuilder +from .feature_normalizer import FeatureNormalizer +from .text_featurizer import TextFeaturizer, SentencePieceFeaturizer diff --git a/athena/data/datasets/__init__.py b/athena/data/datasets/__init__.py new file mode 100644 index 00000000..a13f1067 --- /dev/null +++ b/athena/data/datasets/__init__.py @@ -0,0 +1,16 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" data.datasets """ diff --git a/athena/data/datasets/base.py b/athena/data/datasets/base.py new file mode 100644 index 00000000..acc437df --- /dev/null +++ b/athena/data/datasets/base.py @@ -0,0 +1,146 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li; Shuaijiang Zhao +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" base dataset """ + +import math +import random +from absl import logging +import tensorflow as tf + +from ...utils.data_queue import DataQueue + +def data_loader(dataset_builder, batch_size=16, num_threads=1): + """ dataloader + """ + num_samples = len(dataset_builder) + if num_samples == 0: + raise ValueError("num samples is empty") + + if num_threads == 1: + def _gen_data(): + """ multi thread loader """ + for i in range(num_samples): + yield dataset_builder[i] + else: + # multi-thread + logging.info("loading data using %d threads" % num_threads) + data_queue = DataQueue( + lambda i: dataset_builder[i], + capacity=4096, + num_threads=num_threads, + max_index=num_samples + ) + def _gen_data(): + """ multi thread loader """ + for _ in range(num_samples): + yield data_queue.get() + + # make dataset using from_generator + dataset = tf.compat.v2.data.Dataset.from_generator( + _gen_data, + output_types=dataset_builder.sample_type, + output_shapes=dataset_builder.sample_shape, + ) + + # Padding the features to its max length dimensions. + dataset = dataset.padded_batch( + batch_size=batch_size, + padded_shapes=dataset_builder.sample_shape, + drop_remainder=True, + ) + + # Prefetch to improve speed of input pipeline. + dataset = dataset.prefetch(buffer_size=500) + return dataset + + +class BaseDatasetBuilder: + """ base dataset """ + + def __init__(self): + self.entries = [] + self.speakers = [] + + def __getitem__(self, index): + raise NotImplementedError + + def __len__(self): + return len(self.entries) + + @property + def entries_list(self): + """ return the entries list """ + return self.entries + + @property + def sample_type(self): + """ example types """ + raise NotImplementedError + + @property + def sample_shape(self): + """ examples shapes """ + raise NotImplementedError + + @property + def sample_signature(self): + """ examples signature """ + raise NotImplementedError + + def as_dataset(self, batch_size=16, num_threads=1): + """ return tf.data.Dataset object """ + return data_loader(self, batch_size, num_threads) + + def shard(self, num_shards, index): + """ Creates a Dataset that includes only 1/num_shards of this dataset """ + if index >= num_shards: + raise ValueError("the index should smaller the num_shards") + logging.info("Creates the sub-dataset which is the %d part of %d" % (index, num_shards)) + original_entries = self.entries + self.entries = [] + total_samples = (len(original_entries) // num_shards) * num_shards + for i in range(total_samples): + if i % num_shards == index: + self.entries.append(original_entries[i]) + return self + + def batch_wise_shuffle(self, batch_size=64): + """Batch-wise shuffling of the data entries. + + Each data entry is in the format of (audio_file, file_size, transcript). + If epoch_index is 0 and sortagrad is true, we don't perform shuffling and + return entries in sorted file_size order. Otherwise, do batch_wise shuffling. + + Args: + batch_size: an integer for the batch size. default=64 + """ + if len(self.entries) == 0: + return self + logging.info("perform batch_wise_shuffle with batch_size %d" % batch_size) + max_buckets = int(math.floor(len(self.entries) / batch_size)) + total_buckets = [i for i in range(max_buckets)] + random.shuffle(total_buckets) + shuffled_entries = [] + for i in total_buckets: + shuffled_entries.extend(self.entries[i * batch_size : (i + 1) * batch_size]) + shuffled_entries.extend(self.entries[max_buckets * batch_size :]) + self.entries = shuffled_entries + return self + + # pylint: disable=unused-argument + def compute_cmvn_if_necessary(self, is_necessary=True): + """ vitural interface """ + return self diff --git a/athena/data/datasets/language_set.py b/athena/data/datasets/language_set.py new file mode 100644 index 00000000..eab9e809 --- /dev/null +++ b/athena/data/datasets/language_set.py @@ -0,0 +1,119 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=no-member, invalid-name +""" audio dataset """ +from absl import logging +import tqdm +import tensorflow as tf +from ..text_featurizer import TextFeaturizer +from ...utils.hparam import register_and_parse_hparams +from .base import BaseDatasetBuilder + +class LanguageDatasetBuilder(BaseDatasetBuilder): + """ LanguageDatasetBuilder + """ + default_config = { + "input_text_config": None, + "output_text_config": None, + "input_length_range": [1, 1000], + "output_length_range": [1, 1000] + } + def __init__(self, config=None): + super().__init__() + self.hparams = register_and_parse_hparams(self.default_config, config, cls=self.__class__) + self.input_text_featurizer = TextFeaturizer(self.hparams.input_text_config) + self.output_text_featurizer = TextFeaturizer(self.hparams.output_text_config) + + def load_csv(self, file_path): + """ load csv file """ + logging.info("Loading data from {}".format(file_path)) + with open(file_path, "r", encoding="utf-8") as file: + lines = file.read().splitlines() + lines = lines[1:] + lines = [line.split("\t") for line in lines] + entries = [tuple(line) for line in lines] + input_transcripts, output_transcripts = zip(*entries) + if self.input_text_featurizer.model_type == "text": + self.input_text_featurizer.load_model(input_transcripts) + if self.output_text_featurizer.model_type == "text": + self.output_text_featurizer.load_model(output_transcripts) + self.entries = [] + entries = tqdm.tqdm(entries) + for input_transcript, output_transcript in entries: + input_labels = self.input_text_featurizer.encode(input_transcript) + output_labels = self.output_text_featurizer.encode(output_transcript) + input_length = len(input_labels) + output_length = len(output_labels) + if input_length not in range(self.hparams.input_length_range[0], + self.hparams.input_length_range[1]): + continue + if output_length not in range(self.hparams.output_length_range[0], + self.hparams.output_length_range[1]): + continue + self.entries.append(tuple( + [input_labels, input_length, output_labels, output_length])) + + self.entries.sort(key=lambda item: float(item[1])) + + return self + + def __getitem__(self, index): + input_labels, input_length, output_labels, output_length = self.entries[index] + + return { + "input": input_labels, + "input_length": input_length, + "output": output_labels, + "output_length": output_length, + } + + def __len__(self): + """ return the number of data samples """ + return len(self.entries) + + @property + def num_class(self): + """ return the max_index of the vocabulary """ + return len(self.output_text_featurizer) + + @property + def sample_type(self): + return { + "input": tf.int32, + "input_length": tf.int32, + "output": tf.int32, + "output_length": tf.int32, + } + + @property + def sample_shape(self): + return { + "input": tf.TensorShape([None]), + "input_length": tf.TensorShape([]), + "output": tf.TensorShape([None]), + "output_length": tf.TensorShape([]), + } + + @property + def sample_signature(self): + return ( + { + "input": tf.TensorSpec(shape=(None, None), dtype=tf.int32), + "input_length": tf.TensorSpec(shape=([None]), dtype=tf.int32), + "output": tf.TensorSpec(shape=(None, None), dtype=tf.int32), + "output_length": tf.TensorSpec(shape=([None]), dtype=tf.int32), + }, + ) diff --git a/athena/data/datasets/speech_recognition.py b/athena/data/datasets/speech_recognition.py new file mode 100644 index 00000000..ea7acc40 --- /dev/null +++ b/athena/data/datasets/speech_recognition.py @@ -0,0 +1,268 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li; Shuaijiang Zhao +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=no-member, invalid-name +""" audio dataset """ + +import os +from absl import logging +import tensorflow as tf +from athena.transform import AudioFeaturizer +from ...utils.hparam import register_and_parse_hparams +from ..text_featurizer import TextFeaturizer +from ..feature_normalizer import FeatureNormalizer +from .base import BaseDatasetBuilder + + +class SpeechRecognitionDatasetBuilder(BaseDatasetBuilder): + """ SpeechRecognitionDatasetBuilder + + Args: + for __init__(self, config=None) + + Config:: + audio_config: the config file for feature extractor, default={'type':'Fbank'} + vocab_file: the vocab file, default='data/utils/ch-en.vocab' + + Interfaces:: + __len__(self): return the number of data samples + num_class(self): return the max_index of the vocabulary + 1 + @property: + sample_shape: + {"input": tf.TensorShape([None, self.audio_featurizer.dim, + self.audio_featurizer.num_channels]), + "input_length": tf.TensorShape([1]), + "output_length": tf.TensorShape([1]), + "output": tf.TensorShape([None])} + """ + default_config = { + "audio_config": {"type": "Fbank"}, + "text_config": {"type":"vocab", "model":"athena/utils/vocabs/ch-en.vocab"}, + "cmvn_file": None, + "remove_unk": True, + "input_length_range": [20, 50000], + "output_length_range": [1, 10000], + "speed_permutation": [1.0], + } + + def __init__(self, config=None): + super().__init__() + # hparams + self.hparams = register_and_parse_hparams( + self.default_config, config, cls=self.__class__) + logging.info("hparams: {}".format(self.hparams)) + + self.audio_featurizer = AudioFeaturizer(self.hparams.audio_config) + self.feature_normalizer = FeatureNormalizer(self.hparams.cmvn_file) + self.text_featurizer = TextFeaturizer(self.hparams.text_config) + + def reload_config(self, config): + """ reload the config """ + if config is not None: + self.hparams.override_from_dict(config) + + def preprocess_data(self, file_path): + """ Generate a list of tuples (wav_filename, wav_length_ms, transcript speaker).""" + logging.info("Loading data from {}".format(file_path)) + with open(file_path, "r", encoding="utf-8") as file: + lines = file.read().splitlines() + headers = lines[0] + lines = lines[1:] + lines = [line.split("\t") for line in lines] + self.entries = [tuple(line) for line in lines] + + self.speakers = [] + if "speaker" not in headers.split("\t"): + entries = self.entries + self.entries = [] + if self.text_featurizer.model_type == "text": + _, _, all_transcripts = zip(*entries) + self.text_featurizer.load_model(all_transcripts) + for wav_filename, wav_len, transcripts in entries: + self.entries.append( + tuple([wav_filename, wav_len, transcripts, "global"]) + ) + self.speakers.append("global") + else: + if self.text_featurizer.model_type == "text": + _, _, all_transcripts, _ = zip(*entries) + self.text_featurizer.load_model(all_transcripts) + for _, _, _, speaker in self.entries: + if speaker not in self.speakers: + self.speakers.append(speaker) + + entries = self.entries + self.entries = [] + if len(self.hparams.speed_permutation) > 1: + logging.info("perform speed permutation") + for speed in self.hparams.speed_permutation: + for wav_filename, wav_len, transcripts, speaker in entries: + self.entries.append( + tuple([wav_filename, + float(wav_len) / float(speed), transcripts, speed, speaker + ])) + + self.entries.sort(key=lambda item: float(item[1])) + + # apply some filter + self.filter_sample_by_unk() + self.filter_sample_by_input_length() + self.filter_sample_by_output_length() + return self + + def load_csv(self, file_path): + """ load csv file """ + return self.preprocess_data(file_path) + + def __getitem__(self, index): + audio_data, _, transcripts, speed, speaker = self.entries[index] + feat = self.audio_featurizer(audio_data, speed=speed) + feat = self.feature_normalizer(feat, speaker) + feat_length = feat.shape[0] + + label = self.text_featurizer.encode(transcripts) + label_length = len(label) + return { + "input": feat, + "input_length": feat_length, + "output_length": label_length, + "output": label, + } + + def __len__(self): + """ return the number of data samples """ + return len(self.entries) + + @property + def num_class(self): + """ return the max_index of the vocabulary + 1""" + return len(self.text_featurizer) + + @property + def speaker_list(self): + """ return the speaker list """ + return self.speakers + + @property + def audio_featurizer_func(self): + """ return the audio_featurizer function """ + return self.audio_featurizer + + @property + def sample_type(self): + return { + "input": tf.float32, + "input_length": tf.int32, + "output_length": tf.int32, + "output": tf.int32, + } + + @property + def sample_shape(self): + dim = self.audio_featurizer.dim + nc = self.audio_featurizer.num_channels + return { + "input": tf.TensorShape([None, dim, nc]), + "input_length": tf.TensorShape([]), + "output_length": tf.TensorShape([]), + "output": tf.TensorShape([None]), + } + + @property + def sample_signature(self): + dim = self.audio_featurizer.dim + nc = self.audio_featurizer.num_channels + return ( + { + "input": tf.TensorSpec(shape=(None, None, dim, nc), dtype=tf.float32), + "input_length": tf.TensorSpec(shape=(None), dtype=tf.int32), + "output_length": tf.TensorSpec(shape=(None), dtype=tf.int32), + "output": tf.TensorSpec(shape=(None, None), dtype=tf.int32), + }, + ) + + def filter_sample_by_unk(self): + """filter samples which contain unk + """ + if self.hparams.remove_unk is False: + return self + filter_entries = [] + unk = self.text_featurizer.unk_index + if unk == -1: + return self + for items in self.entries: + if unk not in self.text_featurizer.encode(items[2]): + filter_entries.append(items) + self.entries = filter_entries + return self + + def filter_sample_by_input_length(self): + """filter samples by input length + + The length of filterd samples will be in [min_length, max_length) + + Args: + self.hparams.input_length_range = [min_len, max_len] + min_len: the minimal length(ms) + max_len: the maximal length(ms) + returns: + entries: a filtered list of tuples + (wav_filename, wav_len, transcripts, speed, speaker) + """ + min_len = self.hparams.input_length_range[0] + max_len = self.hparams.input_length_range[1] + filter_entries = [] + for items in self.entries: + if int(items[1]) in range(min_len, max_len): + filter_entries.append(items) + self.entries = filter_entries + return self + + def filter_sample_by_output_length(self): + """filter samples by output length + + The length of filterd samples will be in [min_length, max_length) + + Args: + self.hparams.output_length_range = [min_len, max_len] + min_len: the minimal length + max_len: the maximal length + returns: + entries: a filtered list of tuples + (wav_filename, wav_len, transcripts, speed, speaker) + """ + min_len = self.hparams.output_length_range[0] + max_len = self.hparams.output_length_range[1] + filter_entries = [] + for items in self.entries: + if len(items[2]) in range(min_len, max_len): + filter_entries.append(items) + self.entries = filter_entries + return self + + def compute_cmvn_if_necessary(self, is_necessary=True): + """ compute cmvn file + """ + if not is_necessary: + return self + if os.path.exists(self.hparams.cmvn_file): + return self + feature_dim = self.audio_featurizer.dim * self.audio_featurizer.num_channels + with tf.device("/cpu:0"): + self.feature_normalizer.compute_cmvn( + self.entries, self.speakers, self.audio_featurizer, feature_dim + ) + self.feature_normalizer.save_cmvn() + return self diff --git a/athena/data/datasets/speech_recognition_test.py b/athena/data/datasets/speech_recognition_test.py new file mode 100644 index 00000000..76a6b38f --- /dev/null +++ b/athena/data/datasets/speech_recognition_test.py @@ -0,0 +1,47 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=no-member, invalid-name +""" audio dataset """ +import time +import tqdm +from absl import logging +from athena import SpeechRecognitionDatasetBuilder + +def test(): + ''' test the speed of dataset ''' + data_csv = "/tmp-data/dataset/opensource/hkust/train.csv" + dataset_builder = SpeechRecognitionDatasetBuilder( + { + "audio_config": { + "type": "Fbank", + "filterbank_channel_count": 40, + "sample_rate": 8000, + "local_cmvn": False, + }, + "speed_permutation": [0.9, 1.0], + "vocab_file": "examples/asr/hkust/data/vocab" + } + ) + dataset = dataset_builder.load_csv(data_csv).as_dataset(16, 4) + start = time.time() + for _ in tqdm.tqdm(dataset, total=len(dataset_builder)//16): + pass + logging.info(time.time() - start) + + +if __name__ == '__main__': + logging.set_verbosity(logging.INFO) + test() diff --git a/athena/data/datasets/speech_set.py b/athena/data/datasets/speech_set.py new file mode 100644 index 00000000..e2c31907 --- /dev/null +++ b/athena/data/datasets/speech_set.py @@ -0,0 +1,208 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li; Shuaijiang Zhao +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=no-member, invalid-name +""" audio dataset """ +import os +from absl import logging +import tensorflow as tf +from athena.transform import AudioFeaturizer +from ...utils.hparam import register_and_parse_hparams +from ..feature_normalizer import FeatureNormalizer +from .base import BaseDatasetBuilder + + +class SpeechDatasetBuilder(BaseDatasetBuilder): + """ SpeechDatasetBuilder + + Args: + for __init__(self, config=None) + + Config: + feature_config: the config file for feature extractor, default={'type':'Fbank'} + data_csv: the path for original LDC HKUST, + default='/tmp-data/dataset/opensource/hkust/train.csv' + force_process: force process, if force_process=True, we will re-process the dataset, + if False, we will process only if the out_path is empty. default=False + + Interfaces:: + __len__(self): return the number of data samples + + @property: + sample_shape: + {"input": tf.TensorShape([None, self.audio_featurizer.dim, + self.audio_featurizer.num_channels]), + "input_length": tf.TensorShape([]), + "output_length": tf.TensorShape([]), + "output": tf.TensorShape([None, self.audio_featurizer.dim * + self.audio_featurizer.num_channels]),} + """ + + default_config = { + "audio_config": {"type": "Fbank"}, + "cmvn_file": None, + "input_length_range": [20, 50000] + } + + def __init__(self, config=None): + super().__init__() + # hparams + self.hparams = register_and_parse_hparams( + self.default_config, config, cls=self.__class__) + logging.info("hparams: {}".format(self.hparams)) + + self.audio_featurizer = AudioFeaturizer(self.hparams.audio_config) + self.feature_normalizer = FeatureNormalizer(self.hparams.cmvn_file) + # self.file_path = self.hparams.data_csv + # self.preprocess_data() + + def reload_config(self, config): + """ reload the config """ + if config is not None: + self.hparams.override_from_dict(config) + + def preprocess_data(self, file_path): + """Generate a list of tuples (wav_filename, wav_length_ms, speaker).""" + logging.info("Loading data from {}".format(file_path)) + with open(file_path, "r", encoding='utf-8') as file: + lines = file.read().splitlines() + headers = lines[0] + lines = lines[1:] + + self.entries = [] + for line in lines: + items = line.split("\t") + wav_filename, length, speaker = items[0], items[1], 'global' + if 'speaker' in headers.split("\t"): + speaker = items[-1] + self.entries.append(tuple([wav_filename, length, speaker])) + self.entries.sort(key=lambda item: float(item[1])) + + self.speakers = [] + for _, _, speaker in self.entries: + if speaker not in self.speakers: + self.speakers.append(speaker) + + self.filter_sample_by_input_length() + return self + + def load_csv(self, file_path): + """ load csv file """ + return self.preprocess_data(file_path) + + def __getitem__(self, index): + audio_file, _, speaker = self.entries[index] + feat = self.audio_featurizer(audio_file) + feat = self.feature_normalizer(feat, speaker) + input_data = feat + output_data = tf.reshape( + feat, [-1, self.audio_featurizer.dim * self.audio_featurizer.num_channels] + ) + + return { + "input": input_data, + "input_length": input_data.shape[0], + "output": output_data, + "output_length": output_data.shape[0], + } + + def __len__(self): + """ return the number of data samples """ + return len(self.entries) + + @property + def num_class(self): + """ return the max_index of the vocabulary """ + target_dim = self.audio_featurizer.dim * self.audio_featurizer.num_channels + return target_dim + + @property + def speaker_list(self): + """ return the speaker list """ + return self.speakers + + @property + def audio_featurizer_func(self): + """ return the audio_featurizer function """ + return self.audio_featurizer + + @property + def sample_type(self): + return { + "input": tf.float32, + "input_length": tf.int32, + "output": tf.float32, + "output_length": tf.int32, + } + + @property + def sample_shape(self): + return { + "input": tf.TensorShape( + [None, self.audio_featurizer.dim, self.audio_featurizer.num_channels] + ), + "input_length": tf.TensorShape([]), + "output": tf.TensorShape([None, None]), + "output_length": tf.TensorShape([]), + } + + @property + def sample_signature(self): + return ( + { + "input": tf.TensorSpec( + shape=(None, None, None, None), dtype=tf.float32 + ), + "input_length": tf.TensorSpec(shape=([None]), dtype=tf.int32), + "output": tf.TensorSpec(shape=(None, None, None), dtype=tf.float32), + "output_length": tf.TensorSpec(shape=([None]), dtype=tf.int32), + }, + ) + + def filter_sample_by_input_length(self): + """filter samples by input length + + The length of filterd samples will be in [min_length, max_length) + + Args: + self.hparams.input_length_range = [min_len, max_len] + min_len: the minimal length(ms) + max_len: the maximal length(ms) + returns: + entries: a filtered list of tuples + (wav_filename, wav_len, speaker) + """ + min_len = self.hparams.input_length_range[0] + max_len = self.hparams.input_length_range[1] + filter_entries = [] + for wav_filename, wav_len, speaker in self.entries: + if int(wav_len) in range(min_len, max_len): + filter_entries.append(tuple([wav_filename, wav_len, speaker])) + self.entries = filter_entries + + def compute_cmvn_if_necessary(self, is_necessary=True): + """ compute cmvn file + """ + if not is_necessary: + return self + if os.path.exists(self.hparams.cmvn_file): + return self + feature_dim = self.audio_featurizer.dim * self.audio_featurizer.num_channels + with tf.device("/cpu:0"): + self.feature_normalizer.compute_cmvn( + self.entries, self.speakers, self.audio_featurizer, feature_dim + ) + self.feature_normalizer.save_cmvn() + return self diff --git a/athena/data/feature_normalizer.py b/athena/data/feature_normalizer.py new file mode 100644 index 00000000..2b890235 --- /dev/null +++ b/athena/data/feature_normalizer.py @@ -0,0 +1,116 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li; Shuaijiang Zhao +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=invalid-name +""" Feature Normalizer """ +import os +import json +import tqdm +import time +import pandas +from absl import logging +import tensorflow as tf + + +class FeatureNormalizer: + """ Feature Normalizer """ + + def __init__(self, cmvn_file=None): + super().__init__() + self.cmvn_file = cmvn_file + self.cmvn_dict = {} + self.speakers = [] + if cmvn_file is not None: + self.load_cmvn() + + def __call__(self, feat_date, speaker): + return self.apply_cmvn(feat_date, speaker) + + def apply_cmvn(self, feat_data, speaker): + """ TODO: docstring""" + if speaker not in self.cmvn_dict: + return feat_data + mean = self.cmvn_dict[speaker][0] + var = self.cmvn_dict[speaker][1] + shape = feat_data.get_shape().as_list()[1:] + mean = tf.reshape(tf.convert_to_tensor(mean, dtype=tf.float32), shape) + var = tf.reshape(tf.convert_to_tensor(var, dtype=tf.float32), shape) + feat_data = (feat_data - mean) / tf.sqrt(var) + return feat_data + + def compute_cmvn(self, entries, speakers, featurizer, feature_dim): + """ Compute cmvn for filtered entries """ + start = time.time() + for tar_speaker in speakers: + logging.info("processing %s" % tar_speaker) + initial_mean = tf.Variable(tf.zeros([feature_dim], dtype=tf.float32)) + initial_var = tf.Variable(tf.zeros([feature_dim], dtype=tf.float32)) + total_num = tf.Variable(0, dtype=tf.int32) + + tq_entries = tqdm.tqdm(entries) + for items in tq_entries: + audio_file, speaker = items[0], items[-1] + if speaker != tar_speaker: + continue + feat_data = featurizer(audio_file) + temp_frame_num = feat_data.shape[0] + total_num.assign_add(temp_frame_num) + + temp_feat = tf.reshape(feat_data, [-1, feature_dim]) + temp_feat2 = tf.square(temp_feat) + + temp_mean = tf.reduce_sum(temp_feat, axis=[0]) + temp_var = tf.reduce_sum(temp_feat2, axis=[0]) + + initial_mean.assign_add(temp_mean) + initial_var.assign_add(temp_var) + + # compute mean and var + if total_num == 0: + continue + total_num = tf.cast(total_num, tf.float32) + mean = initial_mean / total_num + variance = initial_var / total_num - tf.square(mean) + self.cmvn_dict[tar_speaker] = (list(mean.numpy()), list(variance.numpy())) + + logging.info("finished compute cmvn, which cost %.4f s" % (time.time() - start)) + + def load_cmvn(self): + """ TODO: docstring """ + if not os.path.exists(self.cmvn_file): + return + cmvns = pandas.read_csv(self.cmvn_file, sep="\t", index_col="speaker") + for speaker, cmvn in cmvns.iterrows(): + self.cmvn_dict[speaker] = ( + json.loads(cmvn["mean"]), + json.loads(cmvn["var"]), + ) + logging.info("Successfully load cmvn file {}".format(self.cmvn_file)) + + def save_cmvn(self): + """ TODO: docstring """ + if self.cmvn_file is None: + self.cmvn_file = "~/.athena/cmvn_file" + cmvn_dir = os.path.dirname(self.cmvn_file) + if not os.path.exists(cmvn_dir): + os.mkdir(cmvn_dir) + cmvns = [] + for speaker in self.cmvn_dict: + cmvns.append( + (speaker, self.cmvn_dict[speaker][0], self.cmvn_dict[speaker][1]) + ) + df = pandas.DataFrame(data=cmvns, columns=["speaker", "mean", "var"]) + df.to_csv(self.cmvn_file, index=False, sep="\t") + logging.info("Successfully save cmvn file {}".format(self.cmvn_file)) diff --git a/athena/data/text_featurizer.py b/athena/data/text_featurizer.py new file mode 100644 index 00000000..dff440e0 --- /dev/null +++ b/athena/data/text_featurizer.py @@ -0,0 +1,201 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li; Shuaijiang Zhao +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=invalid-name +""" Text featurizer """ + +import os +import re +import warnings +from collections import defaultdict +import sentencepiece as spm +import tensorflow as tf +from ..utils.hparam import register_and_parse_hparams + + +class Vocabulary: + """ Vocabulary + + Interface:: + decode: Convert a list of ids to a sentence, with space inserted + encode: Convert a sentence to a list of ids, with special tokens added + """ + + def __init__(self, vocab_file): + """Initialize vocabulary. + Args: + vocab_file: Vocabulary file name. + """ + super().__init__() + if vocab_file is not None: + self.load_model(vocab_file) + + def load_model(self, vocab_file): + """ load model""" + if vocab_file is None or not os.path.exists(vocab_file): + warnings.warn( + "[Warning] the vocab {} is not exists, make sure you are " + "generating it, otherwise you should check it!".format(vocab_file) + ) + + self.stoi = defaultdict(self._default_unk_index) + self.itos = defaultdict(self._default_unk_symbol) + self.space, self.unk = "", "" + self.unk_index, self.max_index = 0, 0 + + with open(vocab_file, "r", encoding="utf-8") as vocab: + for line in vocab: + if line.startswith("#"): + continue + word, index = line.split() + index = int(index) + self.itos[index] = word + self.stoi[word] = index + if word == self.unk: + self.unk_index = index + if index > self.max_index: + self.max_index = index + + # special deal with the space maybe used in English datasets + if self.stoi[self.space] != self.unk_index: + self.stoi[" "] = self.stoi[self.space] + self.itos[self.stoi[self.space]] = " " + + def _default_unk_index(self): + return self.unk_index + + def _default_unk_symbol(self): + return self.unk + + def __len__(self): + return self.max_index + 1 + + def decode(self, ids): + """Convert a list of ids to a sentence.""" + return "".join([self.itos[id] for id in ids]) + + def encode(self, sentence): + """Convert a sentence to a list of ids, with special tokens added.""" + return [self.stoi[token.lower()] for token in list(sentence.strip())] + + def __call__(self, inputs): + if isinstance(inputs, list): + return self.decode(inputs) + elif isinstance(inputs, int): + return self.itos[inputs] + elif isinstance(inputs, str): + return self.encode(inputs) + else: + raise ValueError("unsupported input") + + +class SentencePieceFeaturizer: + """ TODO: docstring """ + + def __init__(self, spm_file): + self.unk_index = 0 + self.sp = spm.SentencePieceProcessor() + if spm_file is not None: + self.sp.Load(spm_file) + + def load_model(self, model_file): + """ load model """ + self.sp.Load(model_file) + + def __len__(self): + return self.sp.GetPieceSize() + + def encode(self, sentence): + """Convert a sentence to a list of ids by sentence piece model""" + sentence = sentence.upper() + return [self.sp.EncodeAsIds(sentence)] + + def decode(self, ids): + """Conver a list of ids to a sentence""" + return self.sp.DecodeIds(ids) + +class TextTokenizer: + """ Text Tokenizer """ + def __init__(self, text=None): + self.tokenizer = tf.keras.preprocessing.text.Tokenizer() + self.text = text + if text is not None: + self.load_model(text) + + def load_model(self, text): + """ load model """ + self.tokenizer.fit_on_texts(text) + + def __len__(self): + return self.tokenizer.num_words + 1 + + def encode(self, texts): + """Convert a sentence to a list of ids, with special tokens added.""" + return self.tokenizer.texts_to_sequences([texts])[0] + + def decode(self, sequences): + """Conver a list of ids to a sentence""" + return self.tokenizer.sequences_to_texts(sequences[0]) + + +class TextFeaturizer: + """ The main text featurizer interface """ + supported_model = { + "vocab": Vocabulary, + "spm": SentencePieceFeaturizer, + "text": TextTokenizer + } + default_config = { + "type": "text", + "model": None, + } + #pylint: disable=dangerous-default-value, no-member + def __init__(self, config=None): + self.p = register_and_parse_hparams(self.default_config, config) + self.model = self.supported_model[self.p.type](self.p.model) + self.punct_tokens = r"＇｛｝［］＼｜｀～＠＃＄％＾＆＊（）" + self.punct_tokens += r"＿＋，。、‘’“”《》？：；【】——~！@" + self.punct_tokens += r"￥%……&（）,.?<>:;\[\]|`\!@#$%^&()+?\"/_-" + + def load_model(self, model_file): + """ load model """ + self.model.load_model(model_file) + + @property + def model_type(self): + """ the model type """ + return self.p.type + + def delete_punct(self, tokens): + """ delete punctuation tokens """ + return re.sub("[{}]".format(self.punct_tokens), "", tokens) + + def __len__(self): + return len(self.model) + + def encode(self, texts): + """Convert a sentence to a list of ids, with special tokens added.""" + return self.model.encode(texts) + + def decode(self, sequences): + """Conver a list of ids to a sentence""" + return self.model.decode(sequences) + + @property + def unk_index(self): + """ return the unk index """ + if self.p.type == "vocab": + return self.model.unk_index + return -1 diff --git a/athena/decode_main.py b/athena/decode_main.py new file mode 100644 index 00000000..83362023 --- /dev/null +++ b/athena/decode_main.py @@ -0,0 +1,53 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support tensorflow 2.0 +# pylint: disable=invalid-name, no-member +r""" a sample implementation of LAS for HKUST """ +import sys +import json +import tensorflow as tf +from absl import logging +from athena import DecoderSolver +from athena.main import ( + parse_config, + build_model_from_jsonfile, + SUPPORTED_DATASET_BUILDER +) + + +def decode(jsonfile): + """ entry point for model decoding, do some preparation work """ + p, model, _, checkpointer, _ = build_model_from_jsonfile(jsonfile, 0) + if "speed_permutation" in p.dataset_config: + p.dataset_config['speed_permutation'] = [1.0] + dataset_builder = SUPPORTED_DATASET_BUILDER[p.dataset_builder](p.dataset_config) + checkpointer.restore_from_best() + solver = DecoderSolver(model, config=p.decode_config) + dataset_builder = dataset_builder.load_csv(p.test_csv).compute_cmvn_if_necessary(True) + solver.decode(dataset_builder.as_dataset(batch_size=1)) + + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + tf.random.set_seed(1) + + jsonfile = sys.argv[1] + with open(jsonfile) as file: + config = json.load(file) + p = parse_config(config) + DecoderSolver.initialize_devices(p.solver_gpu) + decode(jsonfile) diff --git a/athena/horovod_main.py b/athena/horovod_main.py new file mode 100644 index 00000000..a34f18ca --- /dev/null +++ b/athena/horovod_main.py @@ -0,0 +1,38 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support tensorflow 2.0 +# pylint: disable=invalid-name, no-member +r""" a sample implementation of LAS for HKUST """ +import sys +import json +import tensorflow as tf +import horovod.tensorflow as hvd +from absl import logging +from athena import HorovodSolver +from athena.main import parse_config, train + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + tf.random.set_seed(1) + + JSON_FILE = sys.argv[1] + CONFIG = None + with open(JSON_FILE) as f: + CONFIG = json.load(f) + PARAMS = parse_config(CONFIG) + HorovodSolver.initialize_devices() + train(JSON_FILE, HorovodSolver, hvd.size(), hvd.local_rank()) diff --git a/athena/layers/__init__.py b/athena/layers/__init__.py new file mode 100644 index 00000000..0def3344 --- /dev/null +++ b/athena/layers/__init__.py @@ -0,0 +1,16 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" module """ diff --git a/athena/layers/attention.py b/athena/layers/attention.py new file mode 100644 index 00000000..2a7e0a88 --- /dev/null +++ b/athena/layers/attention.py @@ -0,0 +1,341 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=too-few-public-methods, invalid-name +# pylint: disable=too-many-instance-attributes, no-self-use, too-many-arguments + +""" Attention layers. """ + +from absl import logging +import tensorflow as tf + + +class ScaledDotProductAttention(tf.keras.layers.Layer): + """Calculate the attention weights. + q, k, v must have matching leading dimensions. + k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v. + The mask has different shapes depending on its type(padding or look ahead) + but it must be broadcastable for addition. + + Args: + q: query shape == (..., seq_len_q, depth) + k: key shape == (..., seq_len_k, depth) + v: value shape == (..., seq_len_v, depth_v) + mask: Float tensor with shape broadcastable + to (..., seq_len_q, seq_len_k). Defaults to None. + + Returns: + output, attention_weights + """ + + def call(self, q, k, v, mask): + """This is where the layer's logic lives.""" + matmul_qk = tf.matmul(q, k, transpose_b=True) # (..., seq_len_q, seq_len_k) + + # scale matmul_qk + dk = tf.cast(tf.shape(k)[-1], tf.float32) + scaled_attention_logits = matmul_qk / tf.math.sqrt(dk) + + # add the mask to the scaled tensor. + if mask is not None: + scaled_attention_logits += mask * -1e9 + + # softmax is normalized on the last axis (seq_len_k) so that the scores + # add up to 1. + # (..., seq_len_q, seq_len_k) + attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1) + + output = tf.matmul(attention_weights, v) # (..., seq_len_q, depth_v) + + return output, attention_weights + + +class MultiHeadAttention(tf.keras.layers.Layer): + """ Multi-head attention + + Multi-head attention consists of four parts: * Linear layers and split into + heads. * Scaled dot-product attention. * Concatenation of heads. * Final linear layer. + Each multi-head attention block gets three inputs; Q (query), K (key), V (value). + These are put through linear (Dense) layers and split up into multiple heads. + The scaled_dot_product_attention defined above is applied to each head (broadcasted for + efficiency). An appropriate mask must be used in the attention step. The attention + output for each head is then concatenated (using tf.transpose, and tf.reshape) and + put through a final Dense layer. + Instead of one single attention head, Q, K, and V are split into multiple heads because + it allows the model to jointly attend to information at different positions from + different representational spaces. After the split each head has a reduced dimensionality, + so the total computation cost is the same as a single head attention with full + dimensionality. + """ + + def __init__(self, d_model, num_heads): + super().__init__() + self.num_heads = num_heads + self.d_model = d_model + + assert d_model % self.num_heads == 0 + + self.depth = d_model // self.num_heads + + self.wq = tf.keras.layers.Dense( + d_model, + kernel_initializer=tf.compat.v1.truncated_normal_initializer(stddev=0.02), + input_shape=(d_model,), + ) + self.wk = tf.keras.layers.Dense( + d_model, + kernel_initializer=tf.compat.v1.truncated_normal_initializer(stddev=0.02), + input_shape=(d_model,), + ) + self.wv = tf.keras.layers.Dense( + d_model, + kernel_initializer=tf.compat.v1.truncated_normal_initializer(stddev=0.02), + input_shape=(d_model,), + ) + + self.attention = ScaledDotProductAttention() + + self.dense = tf.keras.layers.Dense( + d_model, + kernel_initializer=tf.compat.v1.truncated_normal_initializer(stddev=0.02), + input_shape=(d_model,), + ) + + def split_heads(self, x, batch_size): + """Split the last dimension into (num_heads, depth). + + Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth) + """ + x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth)) + return tf.transpose(x, perm=[0, 2, 1, 3]) + + def call(self, v, k, q, mask): + """ call function """ + batch_size = tf.shape(q)[0] + + q = self.wq(q) # (batch_size, seq_len, hiddn_dim) + k = self.wk(k) # (batch_size, seq_len, hiddn_dim) + v = self.wv(v) # (batch_size, seq_len, hiddn_dim) + + q = self.split_heads(q, batch_size) # (batch_size, num_heads, seq_len_q, depth) + k = self.split_heads(k, batch_size) # (batch_size, num_heads, seq_len_k, depth) + v = self.split_heads(v, batch_size) # (batch_size, num_heads, seq_len_v, depth) + + # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth) + # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k) + scaled_attention, attention_weights = self.attention(q, k, v, mask) + + # (batch_size, seq_len_q, num_heads, depth) + scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3]) + + # (batch_size, seq_len_q, d_model) + concat_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model)) + + output = self.dense(concat_attention) # (batch_size, seq_len_q, d_model) + + return output, attention_weights + + +class BahdanauAttention(tf.keras.Model): + """ the Bahdanau Attention """ + + def __init__(self, units, input_dim=1024): + super().__init__() + self.W1 = tf.keras.layers.Dense( + units, + kernel_initializer=tf.compat.v1.truncated_normal_initializer(stddev=0.02), + input_shape=(input_dim,), + ) + self.W2 = tf.keras.layers.Dense( + units, + kernel_initializer=tf.compat.v1.truncated_normal_initializer(stddev=0.02), + input_shape=(input_dim,), + ) + self.V = tf.keras.layers.Dense( + 1, kernel_initializer=tf.compat.v1.truncated_normal_initializer(stddev=0.02), + input_shape=(units,), + ) + + def call(self, query, values): + """ call function """ + # hidden shape == (batch_size, hidden size) + # hidden_with_time_axis shape == (batch_size, 1, hidden_size) + # we are doing this to perform addition to calculate the score + hidden_with_time_axis = tf.expand_dims(query, 1) # (64, 1, 1024) + + # score shape == (batch_size, max_length, 1) + # we get 1 at the last axis because we are applying score to self.V + # the shape of the tensor before applying self.V is (batch_size, max_length, units) + score = self.V(tf.nn.tanh(self.W1(values) + self.W2(hidden_with_time_axis))) + + # attention_weights shape == (batch_size, max_length, 1) + attention_weights = tf.nn.softmax(score, axis=1) + + # context_vector shape after sum == (batch_size, hidden_size) + context_vector = attention_weights * values + context_vector = tf.reduce_sum(context_vector, axis=1) + + return context_vector, attention_weights + + +class HanAttention(tf.keras.layers.Layer): + """ + Refer to [Hierarchical Attention Networks for Document Classification] + (https://www.cs.cmu.edu/~hovy/papers/16HLT-hierarchical-attention-networks.pdf) + wrap `with tf.variable_scope(name, reuse=tf.AUTO_REUSE):` + Input shape: (Batch size, steps, features) + Output shape: (Batch size, features) + """ + + def __init__( + self, + W_regularizer=None, + u_regularizer=None, + b_regularizer=None, + W_constraint=None, + u_constraint=None, + b_constraint=None, + use_bias=True, + **kwargs + ): + + super().__init__(**kwargs) + self.supports_masking = True + self.init = tf.keras.initializers.get("glorot_uniform") + + self.W_regularizer = tf.keras.regularizers.get(W_regularizer) + self.u_regularizer = tf.keras.regularizers.get(u_regularizer) + self.b_regularizer = tf.keras.regularizers.get(b_regularizer) + + self.W_constraint = tf.keras.constraints.get(W_constraint) + self.u_constraint = tf.keras.constraints.get(u_constraint) + self.b_constraint = tf.keras.constraints.get(b_constraint) + + self.use_bias = use_bias + + def build(self, input_shape): + """ build in keras layer """ + # pylint: disable=attribute-defined-outside-init + assert len(input_shape) == 3 + + self.W = self.add_weight( + name="{}_W".format(self.name), + shape=(int(input_shape[-1]), int(input_shape[-1]),), + initializer=self.init, + regularizer=self.W_regularizer, + constraint=self.W_constraint, + ) + + if self.use_bias: + self.b = self.add_weight( + name="{}_b".format(self.name), + shape=(int(input_shape[-1]),), + initializer="zero", + regularizer=self.b_regularizer, + constraint=self.b_constraint, + ) + + self.attention_context_vector = self.add_weight( + name="{}_att_context_v".format(self.name), + shape=(int(input_shape[-1]),), + initializer=self.init, + regularizer=self.u_regularizer, + constraint=self.u_constraint, + ) + self.built = True + + def call(self, inputs, training=None, mask=None): + """ call function in keras """ + batch_size = tf.shape(inputs)[0] + W_3d = tf.tile(tf.expand_dims(self.W, axis=0), tf.stack([batch_size, 1, 1])) + # [batch_size, steps, features] + input_projection = tf.matmul(inputs, W_3d) + + if self.use_bias: + input_projection += self.b + + input_projection = tf.tanh(input_projection) + + # [batch_size, steps, 1] + similaritys = tf.reduce_sum( + tf.multiply(input_projection, self.attention_context_vector), + axis=2, + keep_dims=True, + ) + + # [batch_size, steps, 1] + if mask is not None: + attention_weights = self._masked_softmax(similaritys, mask, axis=1) + else: + attention_weights = tf.nn.softmax(similaritys, axis=1) + + # [batch_size, features] + attention_output = tf.reduce_sum(tf.multiply(inputs, attention_weights), axis=1) + return attention_output + + # pylint: disable=no-self-use + def compute_output_shape(self, input_shape): + """compute output shape""" + return input_shape[0], input_shape[-1] + + def _masked_softmax(self, logits, mask, axis): + """Compute softmax with input mask.""" + e_logits = tf.exp(logits) + masked_e = tf.multiply(e_logits, mask) + sum_masked_e = tf.reduce_sum(masked_e, axis, keep_dims=True) + ones = tf.ones_like(sum_masked_e) + # pay attention to a situation that if len of mask is zero, + # denominator should be set to 1 + sum_masked_e_safe = tf.where(tf.equal(sum_masked_e, 0), ones, sum_masked_e) + return masked_e / sum_masked_e_safe + + +class MatchAttention(tf.keras.layers.Layer): + """ + Refer to [Learning Natural Language Inference with LSTM] + (https://www.aclweb.org/anthology/N16-1170) + wrap `with tf.variable_scope(name, reuse=tf.AUTO_REUSE):` + Input shape: (Batch size, steps, features) + Output shape: (Batch size, steps, features) + """ + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + logging.info("Initialize MatchAttention {}...".format(self.name)) + self.fc_num_units = config["model"]["net"]["structure"]["fc_num_units"] + self.middle_layer = tf.keras.layers.Dense(self.fc_num_units, activation="tanh") + self.attn = tf.keras.layers.Dense(1) + + # pylint: disable=arguments-differ + def call(self, tensors): + """Attention layer.""" + left, right = tensors + + len_left = left.shape[1] + len_right = right.shape[1] + tensor_left = tf.expand_dims(left, axis=2) + tensor_right = tf.expand_dims(right, axis=1) + tensor_left = tf.tile(tensor_left, [1, 1, len_right, 1]) + tensor_right = tf.tile(tensor_right, [1, len_left, 1, 1]) + tensor_merged = tf.concat([tensor_left, tensor_right], axis=-1) + middle_output = self.middle_layer(tensor_merged) + attn_scores = self.attn(middle_output) + attn_scores = tf.squeeze(attn_scores, axis=3) + exp_attn_scores = tf.exp( + attn_scores - tf.reduce_max(attn_scores, axis=-1, keepdims=True) + ) + exp_sum = tf.reduce_sum(exp_attn_scores, axis=-1, keepdims=True) + attention_weights = exp_attn_scores / exp_sum + return tf.matmul(attention_weights, right) diff --git a/athena/layers/commons.py b/athena/layers/commons.py new file mode 100644 index 00000000..6a2e7ecb --- /dev/null +++ b/athena/layers/commons.py @@ -0,0 +1,113 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=too-few-public-methods, invalid-name +# pylint: disable=no-self-use, missing-function-docstring +"""Utils for common layers.""" + +import tensorflow as tf +from athena.layers.functional import make_positional_encoding, collapse4d, gelu + +from athena.layers.functional import splice + + +class PositionalEncoding(tf.keras.layers.Layer): + """ positional encoding can be used in transformer """ + + def __init__(self, d_model, max_position=800, scale=False): + super().__init__() + self.d_model = d_model + self.scale = scale + self.pos_encoding = make_positional_encoding(max_position, d_model) + + def call(self, x): + """ call function """ + seq_len = tf.shape(x)[1] + if self.scale: + x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32)) + x += self.pos_encoding[:, :seq_len, :] + return x + + +class Collapse4D(tf.keras.layers.Layer): + """ callapse4d can be used in cnn-lstm for speech processing + reshape from [N T D C] -> [N T D*C] + """ + + def call(self, x): + return collapse4d(x) + + +class Gelu(tf.keras.layers.Layer): + """Gaussian Error Linear Unit. + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + Returns: + `x` with the GELU activation applied. + """ + + def call(self, x): + return gelu(x) + + +class TdnnLayer(tf.keras.layers.Layer): + """ An implement of Tdnn Layer + Args: + context: a int of left and right context, or + a list of context indexes, e.g. (-2, 0, 2). + output_dim: the dim of the linear transform + """ + + def __init__(self, context, output_dim, use_bias=False, **kwargs): + super().__init__(**kwargs) + + if hasattr(context, "__iter__"): + self.context_size = len(context) + self.context_list = context + else: + self.context_size = context * 2 + 1 + self.context_list = range(-context, context + 1) + + self.output_dim = output_dim + self.linear = tf.keras.layers.Dense(output_dim, use_bias=use_bias) + + def call(self, x, training=None, mask=None): + x = splice(x, self.context_list) + x = self.linear(x, training=training, mask=mask) + return x + + +SUPPORTED_RNNS = { + "lstm": tf.keras.layers.LSTMCell, + "gru": tf.keras.layers.GRUCell, + "cudnnlstm": tf.keras.layers.LSTMCell, + "cudnngru": tf.keras.layers.GRUCell +} + + +ACTIVATIONS = { + "relu": tf.nn.relu, + "relu6": tf.nn.relu6, + "elu": tf.nn.elu, + "selu": tf.nn.selu, + "gelu": gelu, + "leaky_relu": tf.nn.leaky_relu, + "sigmoid": tf.nn.sigmoid, + "softplus": tf.nn.softplus, + "softsign": tf.nn.softsign, + "tanh": tf.nn.tanh, +} diff --git a/athena/layers/functional.py b/athena/layers/functional.py new file mode 100644 index 00000000..17e7301b --- /dev/null +++ b/athena/layers/functional.py @@ -0,0 +1,109 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=invalid-name +"""Utils for common layers.""" + +import numpy as np +import tensorflow as tf +from ..utils.misc import tensor_shape +from tensorflow.python.framework import ops + + +def make_positional_encoding(position, d_model): + """ generate a postional encoding list """ + + def get_angles(pos, i, d_model): + angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(d_model)) + return pos * angle_rates + + angle_rads = get_angles( + np.arange(position)[:, np.newaxis], np.arange(d_model)[np.newaxis, :], d_model + ) + angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2]) + angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2]) + pos_encoding = angle_rads[np.newaxis, ...] + return tf.cast(pos_encoding, dtype=tf.float32) + + +def collapse4d(x, name=None): + """ reshape from [N T D C] -> [N T D*C] + using tf.shape(x), which generate a tensor instead of x.shape + """ + with ops.name_scope(name, "collapse4d") as name: + shape = tensor_shape(x) + N = shape[0] + T = shape[1] + D = shape[2] + C = shape[3] + DC = D * C + out = tf.reshape(x, [N, T, DC]) + return out + + +def splice(x, context): + """ + Splice a tensor along the last dimension with context. + e.g.: + t = [[[1, 2, 3], + [4, 5, 6], + [7, 8, 9]]] + splice_tensor(t, [0, 1]) = + [[[1, 2, 3, 4, 5, 6], + [4, 5, 6, 7, 8, 9], + [7, 8, 9, 7, 8, 9]]] + + Args: + tensor: a tf.Tensor with shape (B, T, D) a.k.a. (N, H, W) + context: a list of context offsets + + Returns: + spliced tensor with shape (..., D * len(context)) + """ + input_shape = tf.shape(x) + B, T = input_shape[0], input_shape[1] + context_len = len(context) + array = tf.TensorArray(x.dtype, size=context_len) + for idx, offset in enumerate(context): + begin = offset + end = T + offset + if begin < 0: + begin = 0 + sliced = x[:, begin:end, :] + tiled = tf.tile(x[:, 0:1, :], [1, abs(offset), 1]) + final = tf.concat((tiled, sliced), axis=1) + else: + end = T + sliced = x[:, begin:end, :] + tiled = tf.tile(x[:, -1:, :], [1, abs(offset), 1]) + final = tf.concat((sliced, tiled), axis=1) + array = array.write(idx, final) + spliced = array.stack() + spliced = tf.transpose(spliced, (1, 2, 0, 3)) + spliced = tf.reshape(spliced, (B, T, -1)) + return spliced + + +def gelu(x): + """Gaussian Error Linear Unit. + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + Returns: + `x` with the GELU activation applied. + """ + cdf = 0.5 * (1.0 + tf.tanh((np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) + return x * cdf diff --git a/athena/layers/transformer.py b/athena/layers/transformer.py new file mode 100644 index 00000000..cc97a6c9 --- /dev/null +++ b/athena/layers/transformer.py @@ -0,0 +1,379 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# changed from the pytorch transformer implementation +# pylint: disable=invalid-name, too-many-instance-attributes +# pylint: disable=too-few-public-methods, too-many-arguments + +""" the transformer model """ +import tensorflow as tf +from .attention import MultiHeadAttention +from .commons import ACTIVATIONS + + +class Transformer(tf.keras.layers.Layer): + """A transformer model. User is able to modify the attributes as needed. The architecture + is based on the paper "Attention Is All You Need". Ashish Vaswani, Noam Shazeer, + Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Lukasz Kaiser, and + Illia Polosukhin. 2017. Attention is all you need. In Advances in Neural Information + Processing Systems, pages 6000-6010. Users can build the BERT(https://arxiv.org/abs/1810.04805) + model with corresponding parameters. + + Args: + d_model: the number of expected features in the encoder/decoder inputs (default=512). + nhead: the number of heads in the multiheadattention models (default=8). + num_encoder_layers: the number of sub-encoder-layers in the encoder (default=6). + num_decoder_layers: the number of sub-decoder-layers in the decoder (default=6). + dim_feedforward: the dimension of the feedforward network model (default=2048). + dropout: the dropout value (default=0.1). + activation: the activation function of encoder/decoder intermediate layer, relu or gelu + (default=relu). + custom_encoder: custom encoder (default=None). + custom_decoder: custom decoder (default=None). + + Examples:: + >>> transformer_model = Transformer(nhead=16, num_encoder_layers=12) + >>> src = tf.random.normal((10, 32, 512)) + >>> tgt = tf.random.normal((20, 32, 512)) + >>> out = transformer_model(src, tgt) + + Note: A full example to apply nn.Transformer module for the word language model is available in + https://github.com/pytorch/examples/tree/master/word_language_model + """ + + def __init__( + self, + d_model=512, + nhead=8, + num_encoder_layers=6, + num_decoder_layers=6, + dim_feedforward=2048, + dropout=0.1, + activation="gelu", + custom_encoder=None, + custom_decoder=None, + ): + super().__init__() + if custom_encoder is not None: + self.encoder = custom_encoder + else: + encoder_layers = [ + TransformerEncoderLayer( + d_model, nhead, dim_feedforward, dropout, activation + ) + for _ in range(num_encoder_layers) + ] + self.encoder = TransformerEncoder(encoder_layers) + + if custom_decoder is not None: + self.decoder = custom_decoder + else: + decoder_layers = [ + TransformerDecoderLayer( + d_model, nhead, dim_feedforward, dropout, activation + ) + for _ in range(num_decoder_layers) + ] + self.decoder = TransformerDecoder(decoder_layers) + + self.d_model = d_model + self.nhead = nhead + + def call(self, src, tgt, src_mask=None, tgt_mask=None, memory_mask=None, + return_encoder_output=False, training=None): + """Take in and process masked source/target sequences. + + Args: + src: the sequence to the encoder (required). + tgt: the sequence to the decoder (required). + src_mask: the additive mask for the src sequence (optional). + tgt_mask: the additive mask for the tgt sequence (optional). + memory_mask: the additive mask for the encoder output (optional). + src_key_padding_mask: the ByteTensor mask for src keys per batch (optional). + tgt_key_padding_mask: the ByteTensor mask for tgt keys per batch (optional). + memory_key_padding_mask: the ByteTensor mask for memory keys per batch (optional). + + Shape: + - src: :math:`(N, S, E)`. + - tgt: :math:`(N, T, E)`. + - src_mask: :math:`(N, S)`. + - tgt_mask: :math:`(N, T)`. + - memory_mask: :math:`(N, S)`. + + Note: [src/tgt/memory]_mask should be a ByteTensor where True values are positions + that should be masked with float('-inf') and False values will be unchanged. + This mask ensures that no information will be taken from position i if + it is masked, and has a separate mask for each sequence in a batch. + + - output: :math:`(N, T, E)`. + + Note: Due to the multi-head attention architecture in the transformer model, + the output sequence length of a transformer is same as the input sequence + (i.e. target) length of the decode. + + where S is the source sequence length, T is the target sequence length, N is the + batch size, E is the feature number + + Examples: + >>> output = transformer_model(src, tgt, src_mask=src_mask, tgt_mask=tgt_mask) + """ + + if src.shape[0] != tgt.shape[0]: + raise RuntimeError("the batch number of src and tgt must be equal") + + if src.shape[2] != self.d_model or tgt.shape[2] != self.d_model: + raise RuntimeError( + "the feature number of src and tgt must be equal to d_model" + ) + + memory = self.encoder(src, src_mask=src_mask, training=training) + output = self.decoder( + tgt, memory, tgt_mask=tgt_mask, memory_mask=memory_mask, training=training + ) + if return_encoder_output: + return output, memory + return output + + +class TransformerEncoder(tf.keras.layers.Layer): + """TransformerEncoder is a stack of N encoder layers + + Args: + encoder_layer: an instance of the TransformerEncoderLayer() class (required). + num_layers: the number of sub-encoder-layers in the encoder (required). + norm: the layer normalization component (optional). + + Examples:: + >>> encoder_layer = [TransformerEncoderLayer(d_model=512, nhead=8) + >>> for _ in range(num_layers)] + >>> transformer_encoder = TransformerEncoder(encoder_layer) + >>> src = torch.rand(10, 32, 512) + >>> out = transformer_encoder(src) + """ + + def __init__(self, encoder_layers): + super().__init__() + self.layers = encoder_layers + + def call(self, src, src_mask=None, training=None): + """Pass the input through the endocder layers in turn. + + Args: + src: the sequnce to the encoder (required). + mask: the mask for the src sequence (optional). + + Shape: + see the docs in Transformer class. + """ + output = src + for i in range(len(self.layers)): + output = self.layers[i](output, src_mask=src_mask, training=training) + return output + + +class TransformerDecoder(tf.keras.layers.Layer): + """TransformerDecoder is a stack of N decoder layers + + Args: + decoder_layer: an instance of the TransformerDecoderLayer() class (required). + num_layers: the number of sub-decoder-layers in the decoder (required). + norm: the layer normalization component (optional). + + Examples:: + >>> decoder_layer = [TransformerDecoderLayer(d_model=512, nhead=8) + >>> for _ in range(num_layers)] + >>> transformer_decoder = TransformerDecoder(decoder_layer) + >>> memory = torch.rand(10, 32, 512) + >>> tgt = torch.rand(20, 32, 512) + >>> out = transformer_decoder(tgt, memory) + """ + + def __init__(self, decoder_layers): + super().__init__() + self.layers = decoder_layers + + def call(self, tgt, memory, tgt_mask=None, memory_mask=None, training=None): + """Pass the inputs (and mask) through the decoder layer in turn. + + Args: + tgt: the sequence to the decoder (required). + memory: the sequnce from the last layer of the encoder (required). + tgt_mask: the mask for the tgt sequence (optional). + memory_mask: the mask for the memory sequence (optional). + + Shape: + see the docs in Transformer class. + """ + output = tgt + + for i in range(len(self.layers)): + output = self.layers[i]( + output, + memory, + tgt_mask=tgt_mask, + memory_mask=memory_mask, + training=training, + ) + + return output + + +class TransformerEncoderLayer(tf.keras.layers.Layer): + """TransformerEncoderLayer is made up of self-attn and feedforward network. + This standard encoder layer is based on the paper "Attention Is All You Need". + Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, + Lukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. In Advances in + Neural Information Processing Systems, pages 6000-6010. Users may modify or implement + in a different way during application. + + Args: + d_model: the number of expected features in the input (required). + nhead: the number of heads in the multiheadattention models (required). + dim_feedforward: the dimension of the feedforward network model (default=2048). + dropout: the dropout value (default=0.1). + activation: the activation function of intermediate layer, relu or gelu (default=relu). + + Examples:: + >>> encoder_layer = TransformerEncoderLayer(d_model=512, nhead=8) + >>> src = tf.random(10, 32, 512) + >>> out = encoder_layer(src) + """ + + def __init__( + self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="gelu" + ): + super().__init__() + self.self_attn = MultiHeadAttention(d_model, nhead) + # Implementation of Feedforward model + layers = tf.keras.layers + self.ffn = tf.keras.Sequential( + [ + layers.Dense( + dim_feedforward, + activation=ACTIVATIONS[activation], + kernel_initializer=tf.compat.v1.truncated_normal_initializer( + stddev=0.02 + ), + input_shape=(d_model,), + ), + layers.Dropout(dropout, input_shape=(dim_feedforward,)), + layers.Dense( + d_model, + kernel_initializer=tf.compat.v1.truncated_normal_initializer( + stddev=0.02 + ), + input_shape=(dim_feedforward,), + ), + layers.Dropout(dropout, input_shape=(d_model,)), + ] + ) + + self.norm1 = layers.LayerNormalization(epsilon=1e-8, input_shape=(d_model,)) + self.norm2 = layers.LayerNormalization(epsilon=1e-8, input_shape=(d_model,)) + self.dropout = layers.Dropout(dropout, input_shape=(d_model,)) + + def call(self, src, src_mask=None, training=None): + """Pass the input through the endocder layer. + + Args: + src: the sequnce to the encoder layer (required). + mask: the mask for the src sequence (optional). + + Shape: + see the docs in Transformer class. + """ + out = self.self_attn(src, src, src, mask=src_mask)[0] + out = self.norm1(src + self.dropout(out, training=training)) + out = self.norm2(out + self.ffn(out, training=training)) + + return out + + +class TransformerDecoderLayer(tf.keras.layers.Layer): + """TransformerDecoderLayer is made up of self-attn, multi-head-attn and feedforward network. + This standard decoder layer is based on the paper "Attention Is All You Need". + Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, + Lukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. In Advances in + Neural Information Processing Systems, pages 6000-6010. Users may modify or implement + in a different way during application. + + Args: + d_model: the number of expected features in the input (required). + nhead: the number of heads in the multiheadattention models (required). + dim_feedforward: the dimension of the feedforward network model (default=2048). + dropout: the dropout value (default=0.1). + activation: the activation function of intermediate layer, relu or gelu (default=relu). + + Examples:: + >>> decoder_layer = TransformerDecoderLayer(d_model=512, nhead=8) + >>> memory = tf.random(10, 32, 512) + >>> tgt = tf.random(20, 32, 512) + >>> out = decoder_layer(tgt, memory) + """ + + def __init__( + self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="gelu" + ): + super().__init__() + self.attn1 = MultiHeadAttention(d_model, nhead) + self.attn2 = MultiHeadAttention(d_model, nhead) + # Implementation of Feedforward model + layers = tf.keras.layers + self.ffn = tf.keras.Sequential( + [ + layers.Dense( + dim_feedforward, + activation=ACTIVATIONS[activation], + kernel_initializer=tf.compat.v1.truncated_normal_initializer( + stddev=0.02 + ), + input_shape=(d_model,) + ), + layers.Dropout(dropout, input_shape=(dim_feedforward,)), + layers.Dense( + d_model, + kernel_initializer=tf.compat.v1.truncated_normal_initializer( + stddev=0.02 + ), + input_shape=(dim_feedforward,) + ), + layers.Dropout(dropout, input_shape=(d_model,)), + ] + ) + + self.norm1 = layers.LayerNormalization(epsilon=1e-8, input_shape=(d_model,)) + self.norm2 = layers.LayerNormalization(epsilon=1e-8, input_shape=(d_model,)) + self.norm3 = layers.LayerNormalization(epsilon=1e-8, input_shape=(d_model,)) + self.dropout1 = layers.Dropout(dropout, input_shape=(d_model,)) + self.dropout2 = layers.Dropout(dropout, input_shape=(d_model,)) + + def call(self, tgt, memory, tgt_mask=None, memory_mask=None, training=None): + """Pass the inputs (and mask) through the decoder layer. + + Args: + tgt: the sequence to the decoder layer (required). + memory: the sequnce from the last layer of the encoder (required). + tgt_mask: the mask for the tgt sequence (optional). + memory_mask: the mask for the memory sequence (optional). + + Shape: + see the docs in Transformer class. + """ + out = self.attn1(tgt, tgt, tgt, mask=tgt_mask)[0] + out = self.norm1(tgt + self.dropout1(out, training=training)) + out2 = self.attn2(memory, memory, out, mask=memory_mask)[0] + out = self.norm2(out + self.dropout2(out2, training=training)) + out = self.norm3(out + self.ffn(out, training=training)) + return out diff --git a/athena/loss.py b/athena/loss.py new file mode 100644 index 00000000..cc00ba9d --- /dev/null +++ b/athena/loss.py @@ -0,0 +1,99 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support eager mode and TF>=2.0.0 +# pylint: disable=too-few-public-methods +""" some losses """ +import tensorflow as tf +from .utils.misc import insert_eos_in_labels + + +class CTCLoss(tf.keras.losses.Loss): + """ CTC LOSS + CTC LOSS implemented with Tensorflow + """ + + def __init__(self, logits_time_major=False, blank_index=-1, name="CTCLoss"): + super().__init__(name=name) + self.logits_time_major = logits_time_major + self.blank_index = blank_index + self.need_logit_length = True + + def __call__(self, logits, samples, logit_length=None): + assert logit_length is not None + # use v2 ctc_loss + ctc_loss = tf.nn.ctc_loss( + labels=samples["output"], + logits=logits, + logit_length=logit_length, + label_length=samples["output_length"], + logits_time_major=self.logits_time_major, + blank_index=self.blank_index, + ) + return tf.reduce_mean(ctc_loss) + + +class Seq2SeqSparseCategoricalCrossentropy(tf.keras.losses.CategoricalCrossentropy): + """ Seq2SeqSparseCategoricalCrossentropy LOSS + CategoricalCrossentropy calculated at each character for each sequence in a batch + """ + + def __init__(self, num_classes, eos=-1, by_token=False, by_sequence=True, + from_logits=True, label_smoothing=0.0): + super().__init__(from_logits=from_logits, label_smoothing=label_smoothing, reduction="none") + self.by_token = by_token + self.by_sequence = by_sequence + self.num_classes = num_classes + self.eos = num_classes + eos if eos < 0 else eos + + def __call__(self, logits, samples, logit_length=None): + labels = insert_eos_in_labels(samples["output"], self.eos, samples["output_length"]) + mask = tf.math.logical_not(tf.math.equal(labels, 0)) + labels = tf.one_hot(indices=labels, depth=self.num_classes) + seq_len = tf.shape(labels)[1] + logits = logits[:, :seq_len, :] + loss = self.call(labels, logits) + mask = tf.cast(mask, dtype=loss.dtype) + loss *= mask + if self.by_token: + return tf.divide(tf.reduce_sum(loss), tf.reduce_sum(mask)) + if self.by_sequence: + loss = tf.reduce_sum(loss, axis=-1) + return tf.reduce_mean(loss) + + +class MPCLoss(tf.keras.losses.Loss): + """MPC LOSS + L1 loss for each masked acoustic features in a batch + """ + + def __init__(self, name="MPCLoss"): + super().__init__(name=name) + + def __call__(self, logits, samples, logit_length=None): + target = samples["output"] + shape = tf.shape(logits) + target = tf.reshape(target, shape) + loss = target - logits + # mpc mask + mask = tf.cast(tf.math.equal(tf.reshape(samples["input"], shape), 0), loss.dtype) + loss *= mask + # sequence length mask + seq_mask = tf.sequence_mask(logit_length, shape[1], dtype=loss.dtype) + seq_mask = tf.tile(seq_mask[:, :, tf.newaxis], [1, 1, shape[2]]) + loss *= seq_mask + loss = tf.reduce_sum(tf.abs(loss, name="L1_loss"), 2) + loss = tf.reduce_mean(loss) + return loss diff --git a/athena/main.py b/athena/main.py new file mode 100644 index 00000000..25d865ab --- /dev/null +++ b/athena/main.py @@ -0,0 +1,170 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li; Dongwei Jiang; Xiaoning Lei +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support tensorflow 2.0 +# pylint: disable=invalid-name, no-member, wildcard-import, unused-wildcard-import +""" a sample implementation of LAS for HKUST """ +import sys +import json +import tensorflow as tf +from absl import logging +from athena import * + +SUPPORTED_DATASET_BUILDER = { + "speech_recognition_dataset": SpeechRecognitionDatasetBuilder, + "speech_dataset": SpeechDatasetBuilder, + "language_dataset": LanguageDatasetBuilder, +} + +SUPPORTED_MODEL = { + "deep_speech": DeepSpeechModel, + "speech_transformer": SpeechTransformer, + "speech_transformer2": SpeechTransformer2, + "mtl_transformer_ctc": MtlTransformerCtc, + "mpc": MaskedPredictCoding, + "rnnlm": RNNLM +} + +SUPPORTED_OPTIMIZER = { + "warmup_adam": WarmUpAdam, + "expdecay_adam": ExponentialDecayAdam, + "adam": tf.keras.optimizers.Adam, +} + +DEFAULT_CONFIGS = { + "batch_size": 32, + "num_epochs": 20, + "sorta_epoch": 1, + "ckpt": None, + "summary_dir": None, + "solver_gpu": [0], + "solver_config": None, + "model": "speech_transformer", + "num_classes": None, + "model_config": None, + "pretrained_model": None, + "optimizer": "warmup_adam", + "optimizer_config": None, + "dataset_builder": "speech_recognition_dataset", + "dataset_config": None, + "num_data_threads": 1, + "train_csv": None, + "dev_csv": None, + "test_csv": None, + "decode_config": None, +} + +def parse_config(config): + """ parse config """ + p = register_and_parse_hparams(DEFAULT_CONFIGS, config, cls="main") + logging.info("hparams: {}".format(p)) + return p + +def build_model_from_jsonfile(jsonfile, rank=0, pre_run=True): + """ creates model using configurations in json, load from checkpoint + if previous models exist in checkpoint dir + """ + config = None + with open(jsonfile) as file: + config = json.load(file) + p = parse_config(config) + dataset_builder = SUPPORTED_DATASET_BUILDER[p.dataset_builder](p.dataset_config) + + # models + model = SUPPORTED_MODEL[p.model]( + num_classes=p.num_classes + if p.num_classes is not None + else dataset_builder.num_class, + sample_shape=dataset_builder.sample_shape, + config=p.model_config, + ) + optimizer = SUPPORTED_OPTIMIZER[p.optimizer](p.optimizer_config) + checkpointer = Checkpoint( + checkpoint_directory=p.ckpt, + model=model, + optimizer=optimizer, + ) + if pre_run or p.pretrained_model is not None: + # pre_run for lazy initilize in keras + solver = BaseSolver( + model, + optimizer, + sample_signature=dataset_builder.sample_signature + ) + if p.dev_csv is None: + raise ValueError("we currently need a dev_csv for pre-load") + dataset = dataset_builder.load_csv(p.dev_csv).as_dataset(p.batch_size) + solver.evaluate_step(model.prepare_samples(iter(dataset).next())) + if rank == 0: + set_default_summary_writer(p.summary_dir) + return p, model, optimizer, checkpointer, dataset_builder + + +def train(jsonfile, Solver, rank_size=1, rank=0): + """ entry point for model training, implements train loop + + :param jsonfile: json file to read configuration from + :param Solver: an abstract class that implements high-level logic of train, evaluate, decode, etc + :param rank_size: total number of workers, 1 if using single gpu + :param rank: rank of current worker, 0 if using single gpu + """ + p, model, optimizer, checkpointer, dataset_builder \ + = build_model_from_jsonfile(jsonfile, rank) + epoch = checkpointer.save_counter + if p.pretrained_model is not None and epoch == 0: + p2, pretrained_model, _, _, _ \ + = build_model_from_jsonfile(p.pretrained_model, rank) + model.restore_from_pretrained_model(pretrained_model, p2.model) + + # for cmvn + dataset_builder.load_csv(p.train_csv).compute_cmvn_if_necessary(rank == 0) + + # train + solver = Solver( + model, + optimizer, + sample_signature=dataset_builder.sample_signature, + config=p.solver_config, + ) + while epoch < p.num_epochs: + if rank == 0: + logging.info(">>>>> start training in epoch %d" % epoch) + dataset_builder.load_csv(p.train_csv).shard(rank_size, rank) + if epoch >= p.sorta_epoch: + dataset_builder.batch_wise_shuffle(p.batch_size) + dataset = dataset_builder.as_dataset(p.batch_size, p.num_data_threads) + solver.train(dataset) + + if rank == 0: + logging.info(">>>>> start evaluate in epoch %d" % epoch) + dataset = dataset_builder.load_csv(p.dev_csv).as_dataset(p.batch_size, p.num_data_threads) + loss = solver.evaluate(dataset, epoch) + epoch = epoch + 1 + if rank == 0: + checkpointer(loss) + + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + tf.random.set_seed(1) + + JSON_FILE = sys.argv[1] + CONFIG = None + with open(JSON_FILE) as f: + CONFIG = json.load(f) + PARAMS = parse_config(CONFIG) + BaseSolver.initialize_devices(PARAMS.solver_gpu) + train(JSON_FILE, BaseSolver, 1, 0) diff --git a/athena/metrics.py b/athena/metrics.py new file mode 100644 index 00000000..98995e97 --- /dev/null +++ b/athena/metrics.py @@ -0,0 +1,109 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support eager mode and TF>=2.0.0 +# pylint: disable=unused-argument +""" some metrics """ +from absl import logging +import numpy as np +import tensorflow as tf +from .utils.misc import validate_seqs + + +class CharactorAccuracy: + """ CharactorAccuracy + Base class for Word Error Rate calculation + """ + + def __init__(self, name="CharactorAccuracy"): + self.name = name + self.inf = tf.constant(np.inf, dtype=tf.float32) + self.error_count = tf.keras.metrics.Sum(dtype=tf.float32) + self.total_count = tf.keras.metrics.Sum(dtype=tf.float32) + + def reset_states(self): + """reset num_err and num_total to zero""" + self.error_count.reset_states() + self.total_count.reset_states() + + def update_state(self, sparse_predictions, samples, logit_length=None): + """ Accumulate errors and counts """ + validated_label = tf.cast( + tf.sparse.from_dense(samples["output"]), dtype=tf.int64 + ) + labels_counter = tf.cast(tf.shape(validated_label.values)[0], tf.float32) + + num_errs = tf.edit_distance( + sparse_predictions, validated_label, normalize=False + ) + num_errs = tf.reduce_sum(num_errs) + self.error_count(num_errs) + self.total_count(labels_counter) + return num_errs, labels_counter + + def __call__(self, logits, samples, logit_length=None): + return self.update_state(logits, samples, logit_length) + + def result(self): + """ returns word-error-rate calculated as num_err/num_total """ + error_rate = tf.cast( + self.error_count.result() / self.total_count.result(), tf.float32 + ) + if error_rate == self.inf: + return 0.0 + return 1.0 - error_rate + + +class Seq2SeqSparseCategoricalAccuracy(CharactorAccuracy): + """ Seq2SeqSparseCategoricalAccuracy + Inherits CharactorAccuracy and implements Attention accuracy calculation + """ + + def __init__(self, eos, name="Seq2SeqSparseCategoricalAccuracy"): + super().__init__(name=name) + self.eos = eos + + def __call__(self, logits, samples, logit_length=None): + """ Accumulate errors and counts """ + predictions = tf.argmax(logits, axis=2, output_type=tf.int64) + validated_preds, _ = validate_seqs(predictions, self.eos) + + self.update_state(validated_preds, samples, logit_length) + + +class CTCAccuracy(CharactorAccuracy): + """ CTCAccuracy + Inherits CharactorAccuracy and implements CTC accuracy calculation + """ + + def __init__(self, name="CTCAccuracy"): + super().__init__(name=name) + self.need_logit_length = True + + def __call__(self, logits, samples, logit_length=None): + """ Accumulate errors and counts, logit_length is the output length of encoder""" + assert logit_length is not None + with tf.device("/cpu:0"): + # this only implemented in cpu + # ignore if the input length is larger than the output length + if tf.shape(logits)[1] <= tf.shape(samples["output"])[1] + 1: + logging.warning("the length of logits is shorter than that of labels") + else: + decoded, _ = tf.nn.ctc_greedy_decoder( + tf.transpose(logits, [1, 0, 2]), + tf.cast(logit_length, tf.int32), + ) + + self.update_state(decoded[0], samples) diff --git a/athena/models/__init__.py b/athena/models/__init__.py new file mode 100644 index 00000000..bd2f6d3a --- /dev/null +++ b/athena/models/__init__.py @@ -0,0 +1,16 @@ +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" module """ diff --git a/athena/models/base.py b/athena/models/base.py new file mode 100644 index 00000000..8e7e2f16 --- /dev/null +++ b/athena/models/base.py @@ -0,0 +1,74 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support eager mode +# pylint: disable=useless-super-delegation, unused-argument, no-self-use + +""" base model for models """ +from absl import logging +import tensorflow as tf + +class BaseModel(tf.keras.Model): + """Base class for model.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.loss_function = None + self.metric = None + + def call(self, samples, training=None): + """ call model """ + raise NotImplementedError() + + #pylint: disable=not-callable + def get_loss(self, logits, samples, training=None): + """ get loss """ + if self.loss_function is None: + loss = 0.0 + else: + logit_length = self.compute_logit_length(samples) + loss = self.loss_function(logits, samples, logit_length) + if self.metric is None: + metrics = {} + else: + self.metric(logits, samples, logit_length) + metrics = {self.metric.name: self.metric.result()} + return loss, metrics + + def compute_logit_length(self, samples): + """ compute the logit length """ + return samples["input_length"] + + def reset_metrics(self): + """ reset the metrics """ + if self.metric is not None: + self.metric.reset_states() + + def prepare_samples(self, samples): + """ for special data prepare + carefully: do not change the shape of samples + """ + return samples + + def restore_from_pretrained_model(self, pretrained_model, model_type=""): + """ restore from pretrained model + """ + logging.info("restore from pretrained model") + + def decode(self, samples, hparams): + """ decode interface + """ + logging.info("sorry, this model do not support decode") diff --git a/athena/models/customized.py b/athena/models/customized.py new file mode 100644 index 00000000..326b9041 --- /dev/null +++ b/athena/models/customized.py @@ -0,0 +1,67 @@ +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support eager mode and TF>=2.0.0 +# pylint: disable=no-member, invalid-name, relative-beyond-top-level +""" a implementation of customized model """ + +import tensorflow as tf +from absl import logging +from athena import layers as athena_layers +from athena.utils.hparam import register_and_parse_hparams +from .base import BaseModel + + +def build_layer(input_data, layer_config): + """ + Build one or multiple layers from layer configuration. + args: + layer_config: list of multiple layers, or dict of single layer parameters. + returns: a Keras layer. + """ + if isinstance(layer_config, list): + # Recursively build each layer. + output = input_data + for one_layer_config in layer_config: + logging.info(f"Layer conf: {one_layer_config}") + output = build_layer(output, one_layer_config) + return output + + # Build one layer. + for layer_type in layer_config: + layer_args = layer_config[layer_type] + logging.info(f"Final layer config: {layer_type}: {layer_args}") + layer_cls = getattr(athena_layers, layer_type) + layer = layer_cls(**layer_args) + return layer(input_data) + + +class CustomizedModel(BaseModel): + """ a simple customized model """ + default_config = { + "topo": [{}] + } + def __init__(self, num_classes, sample_shape, config=None): + super().__init__() + self.hparams = register_and_parse_hparams(self.default_config, config) + + logging.info(f"Network topology config: {self.hparams.topo}") + input_feature = tf.keras.layers.Input(shape=sample_shape["input"], dtype=tf.float32) + inner = build_layer(input_feature, self.hparams.topo) + inner = tf.keras.layers.Dense(num_classes)(inner) + self.model = tf.keras.Model(inputs=input_feature, outputs=inner) + logging.info(self.model.summary()) + + def call(self, samples, training=None): + return self.model(samples["input"], training=training) diff --git a/athena/models/deep_speech.py b/athena/models/deep_speech.py new file mode 100644 index 00000000..df66d973 --- /dev/null +++ b/athena/models/deep_speech.py @@ -0,0 +1,91 @@ +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support eager mode and TF>=2.0.0 +# pylint: disable=no-member, invalid-name, relative-beyond-top-level +# pylint: disable=too-many-locals, too-many-statements, too-many-arguments, too-many-instance-attributes +""" a implementation of deep speech 2 model can be used as a sample for ctc model """ + +import tensorflow as tf +from absl import logging +from ..utils.hparam import register_and_parse_hparams +from .base import BaseModel +from ..loss import CTCLoss +from ..metrics import CTCAccuracy +from ..layers.commons import SUPPORTED_RNNS + + +class DeepSpeechModel(BaseModel): + """ a sample implementation of CTC model """ + default_config = { + "conv_filters": 256, + "rnn_hidden_size": 1024, + "num_rnn_layers": 6, + "rnn_type": "gru" + } + def __init__(self, num_classes, sample_shape, config=None): + super().__init__() + self.num_classes = num_classes + 1 + self.loss_function = CTCLoss(blank_index=-1) + self.metric = CTCAccuracy() + self.hparams = register_and_parse_hparams(self.default_config, config, cls=self.__class__) + + layers = tf.keras.layers + input_feature = layers.Input(shape=sample_shape["input"], dtype=tf.float32) + inner = layers.Conv2D( + filters=self.hparams.conv_filters, + kernel_size=(41, 11), + strides=(2, 2), + padding="same", + use_bias=False, + )(input_feature) + inner = layers.BatchNormalization()(inner) + inner = tf.nn.relu6(inner) + inner = layers.Conv2D( + filters=self.hparams.conv_filters, + kernel_size=(21, 11), + strides=(2, 1), + padding="same", + use_bias=False, + )(inner) + inner = layers.BatchNormalization()(inner) + inner = tf.nn.relu6(inner) + _, _, dim, channels = inner.get_shape().as_list() + output_dim = dim * channels + inner = layers.Reshape((-1, output_dim))(inner) + rnn_type = self.hparams.rnn_type + rnn_hidden_size = self.hparams.rnn_hidden_size + + for _ in range(self.hparams.num_rnn_layers): + inner = tf.keras.layers.RNN( + cell=[SUPPORTED_RNNS[rnn_type](rnn_hidden_size)], + return_sequences=True + )(inner) + inner = layers.BatchNormalization()(inner) + inner = layers.Dense(rnn_hidden_size, activation=tf.nn.relu6)(inner) + inner = layers.Dense(self.num_classes)(inner) + self.net = tf.keras.Model(inputs=input_feature, outputs=inner) + logging.info(self.net.summary()) + + def call(self, samples, training=None): + """ call function """ + return self.net(samples["input"], training=training) + + def compute_logit_length(self, samples): + """ used for get logit length """ + input_length = tf.cast(samples["input_length"], tf.float32) + logit_length = tf.math.ceil(input_length / 2) + logit_length = tf.math.ceil(logit_length / 2) + logit_length = tf.cast(logit_length, tf.int32) + return logit_length diff --git a/athena/models/masked_pc.py b/athena/models/masked_pc.py new file mode 100644 index 00000000..a4ae7a9e --- /dev/null +++ b/athena/models/masked_pc.py @@ -0,0 +1,209 @@ +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li; Dongwei Jiang; Wubo Li +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support eager mode and TF>=2.0.0 +# pylint: disable=no-member, invalid-name, relative-beyond-top-level +# pylint: disable=too-many-locals, too-many-statements, too-many-arguments, too-many-instance-attributes +r""" an implementations for MPC +""" + +import tensorflow as tf +from .base import BaseModel +from ..utils.hparam import register_and_parse_hparams +from ..layers.commons import PositionalEncoding +from ..layers.transformer import TransformerEncoder, TransformerEncoderLayer +from ..loss import MPCLoss + + +class MaskedPredictCoding(BaseModel): + """ implementation for MPC pretrain model + Args: + num_filters: a int type number, i.e the number of filters in cnn + d_model: a int type number, i.e dimension of model + num_heads: number of heads in transformer + num_encoder_layers: number of layer in encoder + dff: a int type number, i.e dimension of model + rate: rate of dropout layers + chunk_size: number of consecutive masks, i.e 1 or 3 + keep_probability: probability not to be masked + mode: train mode, i.e MPC: pretrain + max_pool_layers: index of max pool layers in encoder, default is -1 + """ + default_config = { + "return_encoder_output": False, + "num_filters": 512, + "d_model": 512, + "num_heads": 8, + "num_encoder_layers": 12, + "dff": 1280, + "rate": 0.1, + "chunk_size": 3, + "keep_probability": 0.85, + "input_dropout_rate": 0.0 + } + + def __init__(self, num_classes, sample_shape, config=None): + super().__init__() + self.downsample_scale = 4 + self.num_classes = num_classes * self.downsample_scale + + # default settings + self.hparams = register_and_parse_hparams(self.default_config, config, cls=self.__class__) + + # MPC loss fuction and metric + _, self.dim, self.num_channels = sample_shape["input"].as_list() + self.loss_function = MPCLoss() + self.metric = tf.keras.metrics.Mean(name="AverageLoss") + + num_filters = self.hparams.num_filters + d_model = self.hparams.d_model + layers = tf.keras.layers + input_features = layers.Input(shape=sample_shape["input"], dtype=tf.float32) + inner = layers.Conv2D( + filters=num_filters, + kernel_size=(3, 3), + strides=(2, 2), + padding="same", + use_bias=False, + )(input_features) + inner = layers.BatchNormalization()(inner) + inner = tf.nn.relu6(inner) + inner = layers.Conv2D( + filters=num_filters, + kernel_size=(3, 3), + strides=(2, 2), + padding="same", + use_bias=False, + )(inner) + inner = layers.BatchNormalization()(inner) + + inner = tf.nn.relu6(inner) + _, _, dim, channels = inner.get_shape().as_list() + output_dim = dim * channels + inner = layers.Reshape((-1, output_dim))(inner) + + inner = layers.Dense(d_model, activation=tf.nn.relu6)(inner) + inner = PositionalEncoding(d_model, scale=False)(inner) + inner = layers.Dropout(self.hparams.rate)(inner) # self.hparams.rate + self.x_net = tf.keras.Model(inputs=input_features, outputs=inner, name="x_net") + print(self.x_net.summary()) + + encoder_layers = [ + TransformerEncoderLayer( + self.hparams.d_model, + self.hparams.num_heads, + self.hparams.dff, + self.hparams.rate, + "gelu", + ) + for _ in range(self.hparams.num_encoder_layers) + ] + self.encoder = TransformerEncoder(encoder_layers) + self.final_layer = layers.Dense(self.num_classes, input_shape=(d_model,)) + self.randomizer = tf.random_uniform_initializer(0, 1) + + def call(self, samples, training: bool = None): + """ used for training + Args: + samples is a dict, including keys: 'input', 'input_length', 'output_length', 'output' + input: acoustic features, Tensor, shape is (batch, time_len, dim, 1), i.e f-bank + Return: + MPC outputs to fit acoustic features + encoder_outputs: Transformer encoder outputs, Tensor, shape is (batch, seqlen, dim) + """ + x0 = tf.nn.dropout(samples["input"], self.hparams.input_dropout_rate) + x = self.x_net(x0, training=training) + x = self.encoder(x, None, training=training) + return self.final_layer(x) + + def get_loss(self, logits, samples, training=None): + """get MPC loss + Args: + logitsdd: MPC output + Return: + MPC L1 loss + """ + logit_length = self.compute_logit_length(samples) + loss = self.loss_function(logits, samples, logit_length) + self.metric.update_state(loss) + metrics = {self.metric.name: self.metric.result()} + return loss, metrics + + def compute_logit_length(self, samples): + input_length = tf.cast(samples["input_length"], tf.float32) + logit_length = tf.math.ceil(input_length / 2) + logit_length = tf.math.ceil(logit_length / 2) + logit_length = tf.cast(logit_length, tf.int32) + return logit_length + + def generate_mpc_mask(self, input_data): + """ generate mask for pretraining + Args: + acoustic features: i.e F-bank + Return: + mask tensor + """ + dtype = input_data.dtype + chunk_size = self.hparams.chunk_size * self.downsample_scale + batch, seq_len, dim, num_channels = input_data.get_shape().as_list() + if (1 - self.hparams.keep_probability) * seq_len <= chunk_size: + chunk_size = self.downsample_scale + num_chunk = tf.cast( + tf.math.ceil( + tf.divide(tf.cast(seq_len, tf.float32), tf.cast(chunk_size, tf.float32)) + ), + tf.int32, + ) + + # generate mask with shape [batch, num_chunk]: 1.0 for keep, 0.0 for masked + ones = tf.ones([batch, num_chunk], dtype=dtype) + zeros = tf.zeros([batch, num_chunk], dtype=dtype) + probability = ones * self.hparams.keep_probability + random = self.randomizer([batch, num_chunk], dtype=dtype) + mask = tf.where(tf.less(random, probability), ones, zeros) + + # change the mask for [batch, seq_len] + mask = tf.tile(mask[:, :, tf.newaxis], [1, 1, chunk_size]) + mask = tf.reshape(mask, [batch, -1])[:, :seq_len] + + # change the mask into masks with the shape [batch, seq_len, dim, num_channels] + masks = tf.tile(mask[:, :, tf.newaxis, tf.newaxis], [1, 1, dim, num_channels]) + return masks + + def prepare_samples(self, samples): + """ for special data prepare + carefully: do not change the shape of samples + """ + mpc_data = samples["input"] + seq_len = ( + tf.math.floordiv(tf.shape(mpc_data)[1], self.downsample_scale) + * self.downsample_scale + ) + mpc_data = mpc_data[:, :seq_len, :, :] + batch_size, seq_len, dim, num_channels = tf.shape(mpc_data) + # input + samples["input"] = mpc_data * self.generate_mpc_mask(mpc_data) + # output + samples["output"] = tf.reshape(mpc_data, [batch_size, seq_len, dim * num_channels]) + # length + input_length = samples["input_length"] + max_input_length = tf.ones_like(input_length) * seq_len + samples["input_length"] = tf.where( + tf.less(input_length, max_input_length), + input_length, + max_input_length + ) + samples["output_length"] = samples["input_length"] + + return samples diff --git a/athena/models/mtl_seq2seq.py b/athena/models/mtl_seq2seq.py new file mode 100644 index 00000000..f9814303 --- /dev/null +++ b/athena/models/mtl_seq2seq.py @@ -0,0 +1,143 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li; Dongwei Jiang; Wubo Li +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support eager mode and TF>=2.0.0 +# pylint: disable=no-member, invalid-name, relative-beyond-top-level +# pylint: disable=too-many-locals, too-many-statements, too-many-arguments, too-many-instance-attributes +""" a implementation of deep speech 2 model can be used as a sample for ctc model """ + +import tensorflow as tf +from tensorflow.keras.layers import Dense +from .base import BaseModel +from ..loss import CTCLoss +from ..metrics import CTCAccuracy +from .speech_transformer import SpeechTransformer, SpeechTransformer2 +from ..utils.hparam import register_and_parse_hparams +from ..tools.beam_search import BeamSearchDecoder +from ..tools.ctc_scorer import CTCPrefixScorer +from ..tools.lm_scorer import NGramScorer + + +class MtlTransformerCtc(BaseModel): + """ In speech recognition, adding CTC loss to Attention-based seq-to-seq model is known to + help convergence. It usually gives better results than using attention alone. + """ + + SUPPORTED_MODEL = { + "speech_transformer": SpeechTransformer, + "speech_transformer2": SpeechTransformer2, + } + + default_config = { + "model": "speech_transformer", + "model_config": {"return_encoder_output": True}, + "mtl_weight": 0.5 + } + + def __init__(self, num_classes, sample_shape, config=None): + super().__init__() + self.num_classes = num_classes + 1 + self.sos = self.num_classes - 1 + self.eos = self.num_classes - 1 + + self.hparams = register_and_parse_hparams(self.default_config, config, cls=self.__class__) + + self.loss_function = CTCLoss(blank_index=-1) + self.metric = CTCAccuracy() + self.model = self.SUPPORTED_MODEL[self.hparams.model]( + num_classes, sample_shape, self.hparams.model_config + ) + self.decoder = Dense(self.num_classes) + self.ctc_logits = None + + def call(self, samples, training=None): + """ call function in keras layers """ + output, encoder_output = self.model(samples, training=training) + self.ctc_logits = self.decoder(encoder_output, training=training) + return output + + def get_loss(self, logits, samples, training=None): + """ get loss used for training """ + logit_length = self.compute_logit_length(samples) + extra_loss = self.loss_function(self.ctc_logits, samples, logit_length) + self.metric(self.ctc_logits, samples, logit_length) + + main_loss, metrics = self.model.get_loss(logits, samples, training=training) + mtl_weight = self.hparams.mtl_weight + loss = mtl_weight * main_loss + (1.0 - mtl_weight) * extra_loss + metrics[self.metric.name] = self.metric.result() + return loss, metrics + + def compute_logit_length(self, samples): + """ compute the logit length """ + return self.model.compute_logit_length(samples) + + def reset_metrics(self): + """ reset the metrics """ + self.metric.reset_states() + self.model.reset_metrics() + + def restore_from_pretrained_model(self, pretrained_model, model_type=""): + """ A more general-purpose interface for pretrained model restoration + + :param pretrained_model: checkpoint path of mpc model + :param model_type: the type of pretrained model to restore + """ + self.model.restore_from_pretrained_model(pretrained_model, model_type) + + def decode(self, samples, hparams): + """ beam search decoding """ + encoder_output, input_mask = self.model.decode(samples, hparams, return_encoder=True) + # init op + last_predictions = tf.ones([1], dtype=tf.int32) * self.sos + history_predictions = tf.TensorArray( + tf.int32, size=1, dynamic_size=True, clear_after_read=False + ) + history_predictions.write(0, last_predictions) + history_predictions = history_predictions.stack() + init_cand_states = [history_predictions] + step = 0 + beam_size = 1 if not hparams.beam_search else hparams.beam_size + beam_search_decoder = BeamSearchDecoder( + self.num_classes, self.sos, self.eos, beam_size=beam_size + ) + beam_search_decoder.build(self.model.time_propagate) + if hparams.beam_search and hparams.ctc_weight != 0: + ctc_scorer = CTCPrefixScorer( + self.eos, + ctc_beam=hparams.beam_size*2, + num_classes=self.num_classes, + ctc_weight=hparams.ctc_weight, + ) + ctc_logits = self.decoder(encoder_output, training=False) + ctc_logits = tf.math.log(tf.nn.softmax(ctc_logits)) + init_cand_states = ctc_scorer.initial_state(init_cand_states, ctc_logits) + beam_search_decoder.add_scorer(ctc_scorer) + if hparams.lm_weight != 0: + if hparams.lm_path is None: + raise ValueError("lm path should not be none") + lm_scorer = NGramScorer( + hparams.lm_path, + self.sos, + self.eos, + self.num_classes, + lm_weight=hparams.lm_weight, + ) + beam_search_decoder.add_scorer(lm_scorer) + predictions = beam_search_decoder( + history_predictions, init_cand_states, step, (encoder_output, input_mask) + ) + return predictions diff --git a/athena/models/rnn_lm.py b/athena/models/rnn_lm.py new file mode 100644 index 00000000..89e7a5eb --- /dev/null +++ b/athena/models/rnn_lm.py @@ -0,0 +1,87 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li; Xiaoning Lei +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support eager mode +# pylint: disable=no-member, invalid-name +""" RNN language model implementation""" + +import tensorflow as tf +from .base import BaseModel +from ..utils.misc import insert_eos_in_labels, insert_sos_in_labels +from ..utils.hparam import register_and_parse_hparams +from ..layers.commons import SUPPORTED_RNNS + +class RNNLM(BaseModel): + """Standard implementation of a RNNLM. Model mainly consists of embeding layer, + rnn layers(with dropout), and the full connection layer, which are all incuded + in self.model_for_rnn + """ + default_config = { + "d_model": 512, # the dim of model + "rnn_type": 'lstm', # the supported rnn type + "num_layer": 2, # the number of rnn layer + "dropout_rate": 0.1, # dropout for model + "sos": -1, # sos can be -1 or -2 + "eos": -1 # eos can be -1 or -2 + } + def __init__(self, num_classes, sample_shape, config=None): + """ config including the params for build lm """ + super(RNNLM, self).__init__() + p = register_and_parse_hparams(self.default_config, config) + self.num_classes = ( + num_classes + 1 + if p.sos == p.eos + else num_classes + 2 + ) + self.sos = self.num_classes + p.sos + self.eos = self.num_classes + p.eos + self.metric = tf.keras.metrics.Mean(name="AverageLoss") + + layers = tf.keras.layers + input_features = layers.Input(shape=sample_shape["output"], dtype=tf.int32) + inner = tf.keras.layers.Embedding(self.num_classes, p.d_model)(input_features) + for _ in range(p.num_layer): + inner = tf.keras.layers.Dropout(p.dropout_rate)(inner) + inner = tf.keras.layers.RNN( + cell=[SUPPORTED_RNNS[p.rnn_type](p.d_model)], + return_sequences=True + )(inner) + inner = tf.keras.layers.Dropout(p.dropout_rate)(inner) + inner = tf.keras.layers.Dense(self.num_classes)(inner) + self.rnnlm = tf.keras.Model(inputs=input_features, outputs=inner) + + def call(self, samples, training: bool = None): + x = insert_sos_in_labels(samples['input'], self.sos) + return self.rnnlm(x, training=training) + + def save_model(self, path): + """ + for saving model and current weight, path is h5 file name, like 'my_model.h5' + usage: + new_model = tf.keras.models.load_model(path) + """ + self.rnnlm.save(path) + + def get_loss(self, logits, samples, training=None): + """ get loss """ + labels = samples['output'] + labels = insert_eos_in_labels(labels, self.eos, samples['output_length']) + labels = tf.one_hot(labels, self.num_classes) + loss = tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits) + n_token = tf.cast(tf.reduce_sum(samples['output_length'] + 1), tf.float32) + self.metric.update_state(loss) + metrics = {self.metric.name: self.metric.result()} + return tf.reduce_sum(loss) / n_token, metrics diff --git a/athena/models/speech_transformer.py b/athena/models/speech_transformer.py new file mode 100644 index 00000000..36457e21 --- /dev/null +++ b/athena/models/speech_transformer.py @@ -0,0 +1,319 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li; Dongwei Jiang; Xiaoning Lei +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support eager mode +# pylint: disable=no-member, invalid-name, relative-beyond-top-level +# pylint: disable=too-many-locals, too-many-statements, too-many-arguments, too-many-instance-attributes + +""" speech transformer implementation""" + +from absl import logging +import tensorflow as tf +from .base import BaseModel +from ..loss import Seq2SeqSparseCategoricalCrossentropy +from ..metrics import Seq2SeqSparseCategoricalAccuracy +from ..utils.misc import generate_square_subsequent_mask, insert_sos_in_labels +from ..layers.commons import PositionalEncoding +from ..layers.transformer import Transformer +from ..utils.hparam import register_and_parse_hparams +from ..tools.beam_search import BeamSearchDecoder +from ..tools.lm_scorer import NGramScorer + + +class SpeechTransformer(BaseModel): + """ Standard implementation of a SpeechTransformer. Model mainly consists of three parts: + the x_net for input preparation, the y_net for output preparation and the transformer itself + """ + default_config = { + "return_encoder_output": False, + "num_filters": 512, + "d_model": 512, + "num_heads": 8, + "num_encoder_layers": 12, + "num_decoder_layers": 6, + "dff": 1280, + "rate": 0.1, + "schedual_sampling_rate": 0.9, + "label_smoothing_rate": 0.0 + } + + def __init__(self, num_classes, sample_shape, config=None): + super().__init__() + self.hparams = register_and_parse_hparams(self.default_config, config, cls=self.__class__) + + self.num_classes = num_classes + 1 + self.sos = self.num_classes - 1 + self.eos = self.num_classes - 1 + ls_rate = self.hparams.label_smoothing_rate + self.loss_function = Seq2SeqSparseCategoricalCrossentropy( + num_classes=self.num_classes, eos=self.eos, label_smoothing=ls_rate + ) + self.metric = Seq2SeqSparseCategoricalAccuracy(eos=self.eos, name="Accuracy") + + # for the x_net + num_filters = self.hparams.num_filters + d_model = self.hparams.d_model + layers = tf.keras.layers + input_features = layers.Input(shape=sample_shape["input"], dtype=tf.float32) + inner = layers.Conv2D( + filters=num_filters, + kernel_size=(3, 3), + strides=(2, 2), + padding="same", + use_bias=False, + )(input_features) + inner = layers.BatchNormalization()(inner) + inner = tf.nn.relu6(inner) + inner = layers.Conv2D( + filters=num_filters, + kernel_size=(3, 3), + strides=(2, 2), + padding="same", + use_bias=False, + )(inner) + inner = layers.BatchNormalization()(inner) + + inner = tf.nn.relu6(inner) + _, _, dim, channels = inner.get_shape().as_list() + output_dim = dim * channels + inner = layers.Reshape((-1, output_dim))(inner) + + inner = layers.Dense(d_model, activation=tf.nn.relu6)(inner) + inner = PositionalEncoding(d_model, scale=False)(inner) + inner = layers.Dropout(self.hparams.rate)(inner) # self.hparams.rate + self.x_net = tf.keras.Model(inputs=input_features, outputs=inner, name="x_net") + print(self.x_net.summary()) + + # y_net for target + input_labels = layers.Input(shape=sample_shape["output"], dtype=tf.int32) + inner = layers.Embedding(self.num_classes, d_model)(input_labels) + inner = PositionalEncoding(d_model, scale=True)(inner) + inner = layers.Dropout(self.hparams.rate)(inner) + self.y_net = tf.keras.Model(inputs=input_labels, outputs=inner, name="y_net") + print(self.y_net.summary()) + + # transformer layer + self.transformer = Transformer( + self.hparams.d_model, + self.hparams.num_heads, + self.hparams.num_encoder_layers, + self.hparams.num_decoder_layers, + self.hparams.dff, + self.hparams.rate, + ) + + # last layer for output + self.final_layer = layers.Dense(self.num_classes, input_shape=(d_model,)) + + # some temp function + self.random_num = tf.random_uniform_initializer(0, 1) + + def call(self, samples, training: bool = None): + x0 = samples["input"] + y0 = insert_sos_in_labels(samples["output"], self.sos) + x = self.x_net(x0, training=training) + y = self.y_net(y0, training=training) + input_length = self.compute_logit_length(samples) + input_mask, output_mask = self._create_masks(x, input_length, y0) + y, encoder_output = self.transformer( + x, + y, + input_mask, + output_mask, + input_mask, + training=training, + return_encoder_output=True, + ) + y = self.final_layer(y) + if self.hparams.return_encoder_output: + return y, encoder_output + return y + + @staticmethod + def _create_masks(x, input_length, y): + r""" Generate a square mask for the sequence. The masked positions are + filled with float(1.0). Unmasked positions are filled with float(0.0). + """ + input_mask, output_mask = None, None + if x is not None: + input_mask = 1.0 - tf.sequence_mask( + input_length, tf.shape(x)[1], dtype=tf.float32 + ) + input_mask = input_mask[:, tf.newaxis, tf.newaxis, :] + input_mask.set_shape([None, None, None, None]) + if y is not None: + output_mask = tf.cast(tf.math.equal(y, 0), tf.float32) + output_mask = output_mask[:, tf.newaxis, tf.newaxis, :] + look_ahead_mask = generate_square_subsequent_mask(tf.shape(y)[1]) + output_mask = tf.maximum(output_mask, look_ahead_mask) + output_mask.set_shape([None, None, None, None]) + return input_mask, output_mask + + def compute_logit_length(self, samples): + """ used for get logit length """ + input_length = tf.cast(samples["input_length"], tf.float32) + logit_length = tf.math.ceil(input_length / 2) + logit_length = tf.math.ceil(logit_length / 2) + logit_length = tf.cast(logit_length, tf.int32) + return logit_length + + def time_propagate(self, history_logits, history_predictions, step, enc_outputs): + """ TODO: doctring + last_predictions: the predictions of last time_step, [beam_size] + history_predictions: the predictions of history from 0 to time_step, + [beam_size, time_steps] + states: (step) + """ + # merge + (encoder_output, memory_mask) = enc_outputs + step = step + 1 + output_mask = generate_square_subsequent_mask(step) + # propagate 1 step + logits = self.y_net(tf.transpose(history_predictions.stack()), training=False) + logits = self.transformer.decoder( + logits, + encoder_output, + tgt_mask=output_mask, + memory_mask=memory_mask, + training=False, + ) + logits = self.final_layer(logits) + logits = logits[:, -1, :] + history_logits = history_logits.write(step - 1, logits) + return logits, history_logits, step + + def decode(self, samples, hparams, return_encoder=False): + """ beam search decoding """ + x0 = samples["input"] + batch = tf.shape(x0)[0] + x = self.x_net(x0, training=False) + input_length = self.compute_logit_length(samples) + input_mask, _ = self._create_masks(x, input_length, None) + encoder_output = self.transformer.encoder(x, input_mask, training=False) + if return_encoder: + return encoder_output, input_mask + # init op + last_predictions = tf.ones([batch], dtype=tf.int32) * self.sos + history_predictions = tf.TensorArray( + tf.int32, size=1, dynamic_size=True, clear_after_read=False + ) + step = 0 + history_predictions.write(0, last_predictions) + history_predictions = history_predictions.stack() + init_cand_states = [history_predictions] + + beam_size = 1 if not hparams.beam_search else hparams.beam_size + beam_search_decoder = BeamSearchDecoder( + self.num_classes, self.sos, self.eos, beam_size=beam_size + ) + beam_search_decoder.build(self.time_propagate) + if hparams.lm_weight != 0: + if hparams.lm_path is None: + raise ValueError("lm path should not be none") + lm_scorer = NGramScorer( + hparams.lm_path, + self.sos, + self.eos, + self.num_classes, + lm_weight=hparams.lm_weight, + ) + beam_search_decoder.add_scorer(lm_scorer) + predictions = beam_search_decoder( + history_predictions, init_cand_states, step, (encoder_output, input_mask) + ) + return predictions + + def restore_from_pretrained_model(self, pretrained_model, model_type=""): + if model_type == "": + return + if model_type == "mpc": + logging.info("loading from pretrained mpc model") + self.x_net = pretrained_model.x_net + self.transformer.encoder = pretrained_model.encoder + elif model_type == "SpeechTransformer": + logging.info("loading from pretrained SpeechTransformer model") + self.x_net = pretrained_model.x_net + self.y_net = pretrained_model.y_net + self.transformer = pretrained_model.transformer + self.final_layer = pretrained_model.final_layer + else: + raise ValueError("NOT SUPPORTED") + + +class SpeechTransformer2(SpeechTransformer): + """ Decoder for SpeechTransformer2 works in time_propagate fashion, it also supports + scheduled sampling """ + + def call(self, samples, training: bool = None): + """ TODO: docstring """ + x0 = samples["input"] + y0 = insert_sos_in_labels(samples["output"], self.sos) + x = self.x_net(x0, training=training) + input_length = self.compute_logit_length(samples) + input_mask, _ = self._create_masks(x, input_length, None) + encoder_output = self.transformer.encoder(x, input_mask, training=training) + + # init op + batch = tf.shape(x)[0] + last_predictions = tf.ones([batch], dtype=tf.int32) * self.sos + history_predictions = tf.TensorArray( + tf.int32, size=1, dynamic_size=True, clear_after_read=False + ) + history_logits = tf.TensorArray( + tf.float32, size=1, dynamic_size=True, clear_after_read=False + ) + step = 0 + eos_list = tf.fill([batch], False) + + # train loop + max_len = tf.shape(y0)[1] + while tf.cast(1, tf.bool): + history_predictions = history_predictions.write(step, last_predictions) + logits, history_logits, step = self.time_propagate( + history_logits, history_predictions, step, (encoder_output, input_mask) + ) + if training: + if step >= max_len: + break + if self.random_num([1]) > self.hparams.schedual_sampling_rate: + last_predictions = tf.argmax(logits, axis=1, output_type=tf.int32) + else: + last_predictions = y0[:, step] + else: + last_predictions = tf.argmax(logits, axis=1, output_type=tf.int32) + eos_list = tf.logical_or(eos_list, last_predictions == self.eos) + if (tf.reduce_sum(tf.cast(eos_list, tf.int32)) >= batch) or ( + step > 100): + break + y = tf.transpose(history_logits.stack(), [1, 0, 2]) + y = self._padding_with_shorter_part(y, max_len) # padding if need + if self.hparams.return_encoder_output: + return y, encoder_output + return y + + @staticmethod + def _padding_with_shorter_part(pre_logits, max_len): + """ decoder may generate result shorter than label length, so we pad it here """ + batch = tf.shape(pre_logits)[0] + pre_len = tf.shape(pre_logits)[1] + out_dim = tf.shape(pre_logits)[2] + if pre_len < max_len: + padding_len = max_len - pre_len + padding_part = tf.zeros( + [batch, padding_len, out_dim], dtype=pre_logits.dtype + ) + pre_logits = tf.concat((pre_logits, padding_part), axis=1) + return pre_logits diff --git a/athena/solver.py b/athena/solver.py new file mode 100644 index 00000000..c24a583d --- /dev/null +++ b/athena/solver.py @@ -0,0 +1,238 @@ +# coding=utf-8 +# Copyright (C) 2019 ATHENA AUTHORS; Xiangang Li; Jianwei Sun; Ruixiong Zhang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=arguments-differ +# pylint: disable=no-member +"""Base class for cross entropy model.""" + +import warnings +import time +import tensorflow as tf +from absl import logging +import horovod.tensorflow as hvd +from .utils import hparam +from .utils.metric_check import MetricChecker +from .utils.misc import validate_seqs +from .metrics import CharactorAccuracy + + +class BaseSolver(tf.keras.Model): + """Base Solver. + """ + default_config = { + "clip_norm": 100.0, + "log_interval": 10, + "enable_tf_function": True + } + def __init__(self, model, optimizer, sample_signature, config=None, **kwargs): + super().__init__(**kwargs) + self.model = model + self.optimizer = optimizer + self.metric_checker = MetricChecker(self.optimizer) + self.sample_signature = sample_signature + + self.hparams = hparam.HParams(cls=self.__class__) + for keys in self.default_config: + self.hparams.add_hparam(keys, self.default_config[keys]) + if config is not None: + self.hparams.override_from_dict(config) + + @staticmethod + def initialize_devices(visible_gpu_idx=None): + """ initialize hvd devices, should be called firstly """ + gpus = tf.config.experimental.list_physical_devices("GPU") + for gpu in gpus: + tf.config.experimental.set_memory_growth(gpu, True) + if gpus is not None: + assert len(gpus) > len(visible_gpu_idx) + for idx in visible_gpu_idx: + tf.config.experimental.set_visible_devices(gpus[idx], "GPU") + + @staticmethod + def clip_by_norm(grads, norm): + """ clip norm using tf.clip_by_norm """ + if norm <= 0: + return grads + grads = [ + None if gradient is None else tf.clip_by_norm(gradient, norm) + for gradient in grads + ] + return grads + + def train_step(self, samples): + """ train the model 1 step """ + with tf.GradientTape() as tape: + logits = self.model(samples, training=True) + loss, metrics = self.model.get_loss(logits, samples, training=True) + grads = tape.gradient(loss, self.model.trainable_variables) + grads = self.clip_by_norm(grads, self.hparams.clip_norm) + self.optimizer.apply_gradients(zip(grads, self.model.trainable_variables)) + return loss, metrics + + def train(self, dataset, total_batches=-1): + """ Update the model in 1 epoch """ + train_step = self.train_step + if self.hparams.enable_tf_function: + logging.info("please be patient, enable tf.function, it takes time ...") + train_step = tf.function(train_step, input_signature=self.sample_signature) + for batch, samples in enumerate(dataset.take(total_batches)): + # train 1 step + samples = self.model.prepare_samples(samples) + loss, metrics = train_step(samples) + if batch % self.hparams.log_interval == 0: + logging.info(self.metric_checker(loss, metrics)) + self.model.reset_metrics() + + def evaluate_step(self, samples): + """ evaluate the model 1 step """ + logits = self.model(samples, training=False) + loss, metrics = self.model.get_loss(logits, samples, training=False) + return loss, metrics + + def evaluate(self, dataset, epoch): + """ evaluate the model """ + loss_metric = tf.keras.metrics.Mean(name="AverageLoss") + loss, metrics = None, None + evaluate_step = self.evaluate_step + if self.hparams.enable_tf_function: + logging.info("please be patient, enable tf.function, it takes time ...") + evaluate_step = tf.function(evaluate_step, input_signature=self.sample_signature) + self.model.reset_metrics() # init metric.result() with 0 + for batch, samples in enumerate(dataset): + samples = self.model.prepare_samples(samples) + loss, metrics = evaluate_step(samples) + if batch % self.hparams.log_interval == 0: + logging.info(self.metric_checker(loss, metrics, -2)) + loss_metric.update_state(loss) + logging.info(self.metric_checker(loss_metric.result(), metrics, evaluate_epoch=epoch)) + self.model.reset_metrics() + return loss_metric.result() + +class HorovodSolver(BaseSolver): + """ A multi-processer solver based on Horovod """ + + @staticmethod + def initialize_devices(visible_gpu_idx=None): + """ initialize hvd devices, should be called firstly """ + if visible_gpu_idx is not None: + warnings.warn("we can not set the visible gpu idx like this") + hvd.init() + gpus = tf.config.experimental.list_physical_devices("GPU") + for gpu in gpus: + tf.config.experimental.set_memory_growth(gpu, True) + if gpus: + tf.config.experimental.set_visible_devices(gpus[hvd.local_rank()], "GPU") + + def train_step(self, samples): + """ train the model 1 step """ + with tf.GradientTape() as tape: + logits = self.model(samples, training=True) + loss, metrics = self.model.get_loss(logits, samples, training=True) + # Horovod: add Horovod Distributed GradientTape. + tape = hvd.DistributedGradientTape(tape) + grads = tape.gradient(loss, self.model.trainable_variables) + grads = self.clip_by_norm(grads, self.hparams.clip_norm) + self.optimizer.apply_gradients(zip(grads, self.model.trainable_variables)) + return loss, metrics + + def train(self, dataset, total_batches=-1): + """ Update the model in 1 epoch """ + train_step = self.train_step + if self.hparams.enable_tf_function: + logging.info("please be patient, enable tf.function, it takes time ...") + train_step = tf.function(train_step, input_signature=self.sample_signature) + for batch, samples in enumerate(dataset.take(total_batches)): + # train 1 step + samples = self.model.prepare_samples(samples) + loss, metrics = train_step(samples) + # Horovod: broadcast initial variable states from rank 0 to all other processes. + # This is necessary to ensure consistent initialization of all workers when + # training is started with random weights or restored from a checkpoint. + # + # Note: broadcast should be done after the first gradient step to ensure optimizer + # initialization. + if batch == 0: + hvd.broadcast_variables(self.model.trainable_variables, root_rank=0) + hvd.broadcast_variables(self.optimizer.variables(), root_rank=0) + if batch % self.hparams.log_interval == 0 and hvd.local_rank() == 0: + logging.info(self.metric_checker(loss, metrics)) + self.model.reset_metrics() + + def evaluate(self, dataset, epoch=0): + """ evaluate the model """ + loss_metric = tf.keras.metrics.Mean(name="AverageLoss") + loss, metrics = None, None + evaluate_step = self.evaluate_step + if self.hparams.enable_tf_function: + logging.info("please be patient, enable tf.function, it takes time ...") + evaluate_step = tf.function(evaluate_step, input_signature=self.sample_signature) + self.model.reset_metrics() + for batch, samples in enumerate(dataset): + samples = self.model.prepare_samples(samples) + loss, metrics = evaluate_step(samples) + if batch % self.hparams.log_interval == 0 and hvd.local_rank() == 0: + logging.info(self.metric_checker(loss, metrics, -2)) + loss_metric.update_state(loss) + if hvd.local_rank() == 0: + logging.info(self.metric_checker(loss_metric.result(), metrics, evaluate_epoch=epoch)) + self.model.reset_metrics() + return loss_metric.result() + + +class DecoderSolver(BaseSolver): + """ DecoderSolver + """ + default_config = { + "beam_search":True, + "beam_size":4, + "ctc_weight":0.0, + "lm_weight":0.1, + "lm_path":"examples/asr/hkust/data/lm.bin" + } + + # pylint: disable=super-init-not-called + def __init__(self, model, config=None): + super().__init__(model, None, None) + self.model = model + self.hparams = hparam.HParams(cls=self.__class__) + for keys in self.default_config: + self.hparams.add_hparam(keys, self.default_config[keys]) + if config is not None: + self.hparams.override_from_dict(config) + + def decode(self, dataset): + """ decode the model """ + if dataset is None: + return + metric = CharactorAccuracy() + for _, samples in enumerate(dataset): + begin = time.time() + samples = self.model.prepare_samples(samples) + predictions = self.model.decode(samples, self.hparams) + validated_preds = validate_seqs(predictions, self.model.eos)[0] + validated_preds = tf.cast(validated_preds, tf.int64) + num_errs, _ = metric.update_state(validated_preds, samples) + reports = ( + "predictions: %s\tlabels: %s\terrs: %d\tavg_acc: %.4f\tsec/iter: %.4f" + % ( + predictions, + samples["output"].numpy(), + num_errs, + metric.result(), + time.time() - begin, + ) + ) + logging.info(reports) + logging.info("decoding finished") diff --git a/athena/tools/__init__.py b/athena/tools/__init__.py new file mode 100644 index 00000000..dce0fe0e --- /dev/null +++ b/athena/tools/__init__.py @@ -0,0 +1,16 @@ +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" tools """ diff --git a/athena/tools/beam_search.py b/athena/tools/beam_search.py new file mode 100644 index 00000000..47e1c902 --- /dev/null +++ b/athena/tools/beam_search.py @@ -0,0 +1,282 @@ +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=invalid-name +""" the beam search decoder layer in encoder-decoder models """ +from collections import namedtuple +import tensorflow as tf + +CandidateHolder = namedtuple( + "CandidateHolder", + ["cand_seqs", "cand_logits", "cand_states", "cand_scores", "cand_parents"], +) + + +class BeamSearchDecoder: + r""" Beam search decoding used in seq2seq decoder layer + This layer is used for evaluation + + Args: + num_syms: the size of the vocab + sos: the representation of the start symbol, should be an id + eos: the representation of the end symbol, should be an id + beam_size: the beam size + """ + + def __init__(self, num_syms, sos, eos, beam_size): + self.num_syms = num_syms + self.sos = sos + self.eos = eos + self.beam_size = beam_size + self.scorers = [] + self.states = [] + self.decoder_one_step = None + + def build(self, decoder_one_step): + """ Allocate the time propagating function of the decoder + Args: + decoder_one_step: the time propagating function of the decoder + """ + self.decoder_one_step = decoder_one_step + + def add_scorer(self, scorer): + """ Add other auxiliary scorers except for the acoustic model + Args: + scorer: the auxiliary scorer like the lm scorer or + the ctc one decoding scorer + """ + self.scorers.append(scorer) + + def beam_search_score(self, candidate_holder, encoder_outputs): + """Call the time propagating function, fetch the acoustic score at the current step + If needed, call the auxiliary scorer and update cand_states in candidate_holder + Args: + candidate_holder: the param cand_seqs and the cand_logits of it is needed + in the transformer decoder to calculate the output. type: CandidateHolder + encoder_outputs: the encoder outputs from the transformer encoder. + type: tuple, (encoder_outputs, input_mask) + """ + cand_logits = tf.TensorArray( + tf.float32, size=0, dynamic_size=True, clear_after_read=False + ) + cand_logits = cand_logits.unstack( + tf.transpose(candidate_holder.cand_logits, [1, 0, 2]) + ) + cand_seqs = tf.TensorArray( + tf.float32, size=0, dynamic_size=True, clear_after_read=False + ) + cand_seqs = cand_seqs.unstack(tf.transpose(candidate_holder.cand_seqs, [1, 0])) + logits, new_cand_logits, states = self.decoder_one_step( + cand_logits, cand_seqs, self.states, encoder_outputs + ) + new_states = candidate_holder.cand_states + self.states = states + cand_scores = tf.expand_dims(candidate_holder.cand_scores, axis=1) + Z = tf.reduce_logsumexp(logits, axis=(1,), keepdims=True) + logprobs = logits - Z + new_scores = logprobs + cand_scores # shape: (cand_num, num_syms) + if self.scorers: + for scorer in self.scorers: + other_scores, new_states = scorer.score(candidate_holder, new_scores) + if other_scores is not None: + new_scores += other_scores + new_cand_logits = tf.transpose(new_cand_logits.stack(), [1, 0, 2]) + return new_scores, new_cand_logits, new_states + + def deal_with_completed( + self, + completed_scores, + completed_seqs, + completed_length, + new_scores, + candidate_holder, + max_seq_len): + """Add the new calculated completed seq with its score to completed seqs + select top beam_size probable completed seqs with these corresponding scores + Args: + completed_scores: the scores of completed_seqs + completed_seqs: historical top beam_size probable completed seqs + completed_length: the length of completed_seqs + new_scores: the current time step scores + candidate_holder: + max_seq_len: the maximum acceptable output length + Returns: + new_completed_scores: new top probable scores + completed_seqs: new top probable completed seqs + completed_length: new top probable seq length + """ + # Add to pool of completed seqs + new_completed_scores = tf.concat( + [completed_scores, new_scores[:, self.eos]], axis=0 + ) + cand_seq_len = tf.shape(candidate_holder.cand_seqs)[1] + eos_tail = tf.fill( + [tf.shape(candidate_holder.cand_seqs)[0], max_seq_len - cand_seq_len], + self.eos, + ) + new_completed_seqs = tf.concat([candidate_holder.cand_seqs, eos_tail], axis=1) + completed_seqs = tf.concat([completed_seqs, new_completed_seqs], axis=0) + new_completed_length = tf.fill( + [tf.shape(new_completed_seqs)[0]], cand_seq_len + 1 + ) + completed_length = tf.concat([completed_length, new_completed_length], axis=0) + + completed_len = tf.shape(new_completed_scores)[0] + if completed_len > self.beam_size: + # Rescale scores by sequence length + completed_length_float = tf.cast(completed_length, tf.float32) + rescaled_scores = new_completed_scores / completed_length_float + _, inds = tf.math.top_k(rescaled_scores, k=self.beam_size) + new_completed_scores = tf.gather(new_completed_scores, inds) + completed_seqs = tf.gather(completed_seqs, inds) + completed_length = tf.gather(completed_length, inds) + return new_completed_scores, completed_seqs, completed_length + + def deal_with_uncompleted( + self, + new_scores, + new_cand_logits, + new_states, + candidate_holder): + """select top probable candidate seqs from new predictions with its scores + update candidate_holder based on top probable candidates + Args: + new_scores: the current time step prediction scores + new_cand_logits: historical prediction scores + new_states: updated states + candidate_holder: + Returns: + candidate_holder: cand_seqs, cand_logits, cand_states, + cand_scores, cand_parents will be updated here and sent + to next time step + """ + cand_seqs = candidate_holder.cand_seqs + num_cands = tf.shape(candidate_holder.cand_seqs)[0] + + # Deal with non-completed candidates + not_eos = tf.range(self.num_syms) != self.eos + not_eos = tf.range(self.num_syms)[not_eos] + parents, syms = tf.meshgrid( + tf.range(num_cands), tf.range(self.num_syms), indexing="ij" + ) + new_scores = tf.gather(new_scores, not_eos, axis=1) + parents = tf.gather(parents, not_eos, axis=1) + syms = tf.gather(syms, not_eos, axis=1) + new_scores_flat = tf.reshape(new_scores, [-1]) + parents_flat = tf.reshape(parents, [-1]) + syms_flat = tf.reshape(syms, [-1]) + + new_scores_flat_len = tf.shape(new_scores_flat)[0] + if new_scores_flat_len > self.beam_size: + _, inds = tf.math.top_k(new_scores_flat, k=self.beam_size) + else: + inds = tf.range(new_scores_flat_len) + cand_syms = tf.gather(syms_flat, inds) + cand_parents = tf.gather(parents_flat, inds) + # Update beam state + cand_scores = tf.gather(new_scores_flat, inds) + cand_logits = tf.gather(new_cand_logits, cand_parents) + cand_states = [tf.gather(state, cand_parents) for state in new_states] + cand_seqs = tf.gather(cand_seqs, cand_parents) + cand_syms = tf.expand_dims(cand_syms, 1) + cand_seqs = tf.concat([cand_seqs, cand_syms], axis=1) + cand_states[0] = cand_seqs + candidate_holder = CandidateHolder( + cand_seqs, cand_logits, cand_states, cand_scores, cand_parents + ) + return candidate_holder + + def __call__(self, cand_seqs, cand_states, init_states, encoder_outputs): + """ + Args: + cand_seqs: TensorArray list, element shape: [beam] + cand_states: [history_predictions] + init_states: state list + encoder_outputs: (encoder_outputs, memory_mask, ...) + Returns: + completed_seqs: the sequence with highest score + """ + cand_logits = tf.fill([tf.shape(cand_seqs)[0], 0, self.num_syms], 0.0) + cand_scores = tf.fill([tf.shape(cand_seqs)[0]], 0.0) + cand_scores = tf.cast(cand_scores, tf.float32) + cand_parents = tf.fill([1], 0) + candidate_holder = CandidateHolder( + cand_seqs, cand_logits, cand_states, cand_scores, cand_parents + ) + self.states = init_states + + max_seq_len = encoder_outputs[0].shape[1] + completed_seqs = tf.fill([0, max_seq_len], self.eos) + completed_length = tf.fill([0], 1) + completed_scores = tf.fill([0], 0.0) + for pos in tf.range(max_seq_len): + # compute new scores + new_scores, new_cand_logits, new_states = self.beam_search_score( + candidate_holder, encoder_outputs + ) + # extract seqs with end symbol + ( + completed_scores, + completed_seqs, + completed_length, + ) = self.deal_with_completed( + completed_scores, + completed_seqs, + completed_length, + new_scores, + candidate_holder, + max_seq_len, + ) + if (pos + 1) >= max_seq_len: + break + not_eos = tf.range(self.num_syms) != self.eos + not_eos = tf.range(self.num_syms)[not_eos] + new_scores = tf.gather(new_scores, not_eos, axis=1) + # Terminate if all candidates are already worse than all completed seqs + min_completed_score = tf.reduce_min(completed_scores) + max_new_score = tf.reduce_max(new_scores) + cand_seq_len = tf.shape(candidate_holder.cand_seqs)[1] + 1 + cand_seq_len_float = tf.cast(cand_seq_len, tf.float32) + new_scores_rescale = new_scores / cand_seq_len_float + max_new_score_rescale = tf.reduce_max(new_scores_rescale) + completed_length_float = tf.cast(completed_length, tf.float32) + rescale_scores = completed_scores / completed_length_float + min_completed_score_rescale = tf.reduce_min(rescale_scores) + # Strict the statement of termination by scores and normalized scores(by length) + if ( + max_new_score < min_completed_score + and max_new_score_rescale < min_completed_score_rescale + ): + break + candidate_holder = self.deal_with_uncompleted( + new_scores, new_cand_logits, new_states, candidate_holder + ) + encoder_output, input_mask = encoder_outputs + encoder_output = tf.tile( + encoder_output[:1], [tf.shape(candidate_holder.cand_seqs)[0], 1, 1] + ) + input_mask = tf.tile( + input_mask[:1], [tf.shape(candidate_holder.cand_seqs)[0], 1, 1, 1] + ) + encoder_outputs = (encoder_output, input_mask) + + # Sort completed seqs + completed_length_float = tf.cast(completed_length, tf.float32) + rescaled_scores = completed_scores / completed_length_float + _, inds = tf.math.top_k(rescaled_scores, k=1) + length = completed_length[inds[0]] + return tf.cast( + tf.expand_dims(completed_seqs[inds[0]][1:length], axis=0), tf.int64 + ) diff --git a/athena/tools/ctc_scorer.py b/athena/tools/ctc_scorer.py new file mode 100644 index 00000000..fa89bfd1 --- /dev/null +++ b/athena/tools/ctc_scorer.py @@ -0,0 +1,168 @@ +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=invalid-name +import numpy as np +import tensorflow as tf + + +class CTCPrefixScorer: + """ + ctc one pass decoding, the algorithm is based on + "HYBRID CTC/ATTENTION ARCHITECTURE FOR END-TO-END SPEECH RECOGNITION," + """ + + def __init__(self, eos, ctc_beam, num_classes, blank=-1, ctc_weight=0.25): + self.logzero = -10000000000.0 + self.eos = eos + self.blank = blank + self.ctc_beam = ctc_beam + self.state_index = 0 + self.score_index = 0 + self.ctc_weight = ctc_weight + self.num_classes = num_classes + self.x = None + self.input_length = None + self.init_state = None + + def initial_state(self, init_cand_states, x): + """ + Initialize states and Add init_state and init_score to init_cand_states + Args: + init_cand_states: CandidateHolder.cand_states + x: log softmax value from ctc_logits, shape: (beam, T, num_classes) + Return: init_cand_states + """ + self.x = x[0].numpy() + self.input_length = len(self.x) + r = np.full((self.input_length, 2), self.logzero, dtype=np.float32) + r[0, 1] = self.x[0, self.blank] + for i in tf.range(1, self.input_length): + r[i, 1] = r[i - 1, 1] + self.x[i, self.blank] + self.init_state = r + init_cand_states.append( + tf.convert_to_tensor(np.array([[r] * self.num_classes])) + ) + self.state_index = len(init_cand_states) - 1 + init_cand_states.append( + tf.convert_to_tensor(np.array([[0] * self.num_classes])) + ) + self.score_index = len(init_cand_states) - 1 + return init_cand_states + + def score(self, candidate_holder, new_scores): + """ + Call this function to compute the ctc one pass decoding score based on + the logits of the ctc module, the scoring function shares a common interface + Args: + candidate_holder: CandidateHolder + new_scores: the score from other models + Return: + ctc_score_result: shape: (beam, num_classes) + cand_states: CandidateHolder.cand_states updated cand_states + """ + cand_seqs = candidate_holder.cand_seqs + cand_states = candidate_holder.cand_states + cand_syms = cand_seqs[:, -1] + + cand_state_value = [] + cand_score_value = [] + for j in range(cand_states[self.state_index].shape[0]): + cand_state_value.append(cand_states[self.state_index][j][cand_syms[j]]) + cand_score_value.append(cand_states[self.score_index][j][cand_syms[j]]) + ctc_score_result = [] + ctc_score_total = [] + new_states = [] + for i in tf.range(new_scores.shape[0]): + num_sym_state = np.array([self.init_state] * self.num_classes) + num_sym_score = np.array([0.0] * self.num_classes, dtype=np.float32) + num_sym_score_minus = np.array([0.0] * self.num_classes, dtype=np.float32) + cand_seq = cand_seqs[i] + ctc_pre_state = cand_state_value[i] + top_ctc_candidates = np.argsort(new_scores[i, :]) + top_ctc_candidates = sorted(top_ctc_candidates[-self.ctc_beam :].tolist()) + cand_seq = np.array(cand_seq) + top_ctc_candidates = np.array(top_ctc_candidates) + ctc_pre_state = ctc_pre_state.numpy() + ctc_score, new_state = self.cand_score( + cand_seq, top_ctc_candidates, ctc_pre_state + ) + ctc_pre_score = tf.cast(cand_score_value[i], tf.float32) + ctc_score_minus = self.ctc_weight * (ctc_score - ctc_pre_score) + 500 + + for k in range(len(top_ctc_candidates)): + num_sym_score[top_ctc_candidates[k]] = ctc_score[k] + num_sym_score_minus[top_ctc_candidates[k]] = ctc_score_minus[k] + num_sym_state[top_ctc_candidates[k]] = new_state[k] + num_sym_score_minus -= 500 + ctc_score_result.append(num_sym_score_minus) + ctc_score_total.append(num_sym_score) + new_states.append(num_sym_state) + cand_states[self.state_index] = tf.convert_to_tensor(np.array(new_states)) + ctc_score_result = tf.convert_to_tensor(np.array(ctc_score_result)) + ctc_score_total = tf.convert_to_tensor(np.array(ctc_score_total)) + cand_states[self.score_index] = ctc_score_total + return ctc_score_result, cand_states + + def cand_score(self, y, cs, r_prev): + """ + r: the probability of the output seq containing the predicted label + given the current input seqs, shape: [input_length, 2, ctc_beam] + r[:, 0]: the prediction of the t-th frame is not blank + r[:, 1]: the prediction of the t-th frame is blank + log_phi: the probability that the last predicted label is not created + by the t-th frame + log_psi: the sum of all log_phi's, the prefix probability, shape:[ctc_beam] + + Args: + y: cand_seq + cs: top_ctc_candidates + r_prev: ctc_pre_state + Return: + log_psi: ctc_score + new_state + """ + output_length = len(y) - 1 # ignore sos + + r = np.ndarray((self.input_length, 2, len(cs)), dtype=np.float32) + xs = self.x[:, cs] + if output_length == 0: + r[0, 0] = xs[0] + r[0, 1] = self.logzero + else: + # output length larger than input length is not supported + r[output_length - 1] = self.logzero + # r_sum: creating (t-1) labels, shape:[input_length] + r_sum = np.logaddexp(r_prev[:, 0], r_prev[:, 1]) + last = y[-1] + if output_length > 0 and last in cs: + log_phi = np.ndarray((self.input_length, len(cs)), dtype=np.float32) + for i in tf.range(len(cs)): + log_phi[:, i] = r_sum if cs[i] != last else r_prev[:, 1] + else: + log_phi = r_sum + + start = max(output_length, 1) + log_psi = r[start - 1, 0] + for t in tf.range(start, self.input_length): + r[t, 0] = np.logaddexp(r[t - 1, 0], log_phi[t - 1]) + xs[t] + r[t, 1] = np.logaddexp(r[t - 1, 0], r[t - 1, 1]) + self.x[t, self.blank] + log_psi = np.logaddexp(log_psi, log_phi[t - 1] + xs[t]) + + eos_pos = np.where(cs == self.eos)[0] + if len(eos_pos) > 0: + log_psi[eos_pos] = r_sum[-1] + + return log_psi, np.rollaxis(r, 2) diff --git a/athena/tools/lm_scorer.py b/athena/tools/lm_scorer.py new file mode 100644 index 00000000..2c60587d --- /dev/null +++ b/athena/tools/lm_scorer.py @@ -0,0 +1,79 @@ +import numpy as np +import kenlm + + +class NGramScorer(object): + """ + KenLM language model + """ + + def __init__(self, lm_path, sos, eos, num_syms, lm_weight=0.1): + """ + Basic params will be initialized, the kenlm model will be created from + the lm_path + Args: + lm_path: the saved lm model path + sos: start symbol + eos: end symbol + num_syms: number of classes + lm_weight: the lm weight + """ + self.lang_model = kenlm.Model(lm_path) + self.state_index = 0 + self.sos = sos + self.eos = eos + self.num_syms = num_syms + self.lm_weight = lm_weight + kenlm_state = kenlm.State() + self.lang_model.BeginSentenceWrite(kenlm_state) + self.cand_kenlm_states = np.array([[kenlm_state] * num_syms]) + + def score(self, candidate_holder, new_scores): + """ + Call this function to compute the NGram score of the next prediction + based on historical predictions, the scoring function shares a common interface + Args: + candidate_holder: + Returns: + score: the NGram weighted score + cand_states: + """ + cand_seqs = candidate_holder.cand_seqs + cand_parents = candidate_holder.cand_parents + cand_syms = cand_seqs[:, -1] + score = self.get_score(cand_parents, cand_syms, self.lang_model) + score = self.lm_weight * score + return score, candidate_holder.cand_states + + def get_score(self, cand_parents, cand_syms, lang_model): + """ + the saved lm model will be called here + Args: + cand_parents: last selected top candidates + cand_syms: last selected top char index + lang_model: the language model + Return: + scores: the lm scores + """ + scale = 1.0 / np.log10(np.e) # convert log10 to ln + + num_cands = len(cand_syms) + scores = np.zeros((num_cands, self.num_syms)) + new_states = np.zeros((num_cands, self.num_syms), dtype=object) + chars = [str(x) for x in range(self.num_syms)] + chars[self.sos] = "" + chars[self.eos] = "" + chars[0] = "" + + for i in range(num_cands): + parent = cand_parents[i] + kenlm_state_list = self.cand_kenlm_states[parent] + kenlm_state = kenlm_state_list[cand_syms[i]] + for sym in range(self.num_syms): + char = chars[sym] + out_state = kenlm.State() + score = scale * lang_model.BaseScore(kenlm_state, char, out_state) + scores[i, sym] = score + new_states[i, sym] = out_state + self.cand_kenlm_states = new_states + return scores diff --git a/athena/transform/README.md b/athena/transform/README.md new file mode 100644 index 00000000..16c7637b --- /dev/null +++ b/athena/transform/README.md @@ -0,0 +1,227 @@ +# speech featurizer + +## 1. Read wave + +Read audio sample from wav file, return sample data and sample rate. + +### 1.1 read_wav.py + +#### 1.1.1 Usage +```python +from athena.transform.feats.readwav import Readwav +conf = {'audio_channels': 1} +speed = 0.9 +readwav = Readwav.params(conf).instantiate() +audio_data, sample_rate = readwav(filepath, speed) +``` + +#### 1.1.2 Configures Setting[Options] + +```python +"audio_channels" : Number of sample channels wanted. (int, default = 1) +``` + +### 1.2 write_wav.py +#### 1.2.1 Usage + +```python +from athena.transform.feats.write_wav import WriteWav +with tf.cached_session() as sess: + write_wav = WriteWav.params().instantiate() + writewav_op = write_wav(new_path, input_data, sample_rate) + sess.run(writewav_op) +``` + +#### 1.2.2 Configures Setting[Options] + +```python +"sample_rate" : Sample frequency of waveform data. (int, default = 16000) +``` + +## 2. Spectrum + +Audio signal gets sliced into (overlapping) frames and a window function is applied to each frame. +Afterwards, we do a Fourier transform on each frame (or more specifically a Short-Time Fourier Transform) and calculate the power spectrum. + +### 2.1 spectrum.py + +#### 2.1.1 Usage + +```python +from athena.transform.feats.spectrum import Spectrum +conf = {'window_length':0.025, 'output_type':1} +spectrum = Spectrum.params(conf).instantiate() +spectrum_feats = spectrum(input_data, sample_rate) +``` + +#### 2.1.2 Configures Setting [Options] + +```python +"window_length" : "Window length in seconds. (float, default = 0.025)" +"frame_length" : "Hop length in seconds. (float, default = 0.010)" +"snip_edges" : "If 1, the last frame (shorter than window_length) will be cutoff. If 2, 1 // 2 frame_length data will be padded to data. (int, default = 1)" +"raw_energy" : "If 1, compute frame energy before preemphasis and windowing. If 2, compute frame energy after preemphasis and windowing. (int, default = 1)" +"preEph_coeff" : "Coefficient for use in frame-signal preemphasis. (float, default = 0.97)" +"window_type" : "Type of window ('hamm'|'hann'|'povey'|'rect'|'blac'|'tria'). (string, default='povey')" +"remove_dc_offset" : "Subtract mean from waveform on each frame (bool, default = true)" +"is_fbank" : "If true, compute power spetrum without frame energy. If false, using the frame energy instead of the square of the constant component of the signal. (bool, default = false)" +"output_type" : "If 1, return power spectrum. If 2, return log-power spectrum. (int, default = 2)" +"dither" : "Dithering constant (0.0 means no dither) (float, default = 1) [add robust to training]" +``` + +## 3. FilterBank + +Computing filter banks is applying triangular filters on a Mel-scale to the power spectrum to extract frequency bands. + +### 3.1 fbank.py + +#### 3.1.1 Usage + +```python +from athena.transform.feats.fbank import Fbank +conf = {'delta_delta':False, 'window_length':0.025, 'output_type':1} +fbank = Fbank.params(conf).instantiate() +fbank_feats = fbank(input_data, sample_rate) +``` + +#### 3.1.2 Configures Setting [Options] + +```python +"window_length" : "Window length in seconds. (float, default = 0.025)" +"frame_length" : "Hop length in seconds. (float, default = 0.010)" +"snip_edges" : "If 1, the last frame (shorter than window_length) will be cutoff. If 2, 1 // 2 frame_length data will be padded to data. (int, default = 1)" +"raw_energy" : "If 1, compute frame energy before preemphasis and windowing. If 2, compute frame energy after preemphasis and windowing. (int, default = 1)" +"preEph_coeff" : "Coefficient for use in frame-signal preemphasis. (float, default = 0.97)" +"window_type" : "Type of window ('hamm'|'hann'|'povey'|'rect'|'blac'|'tria'). (string, default='povey')" +"remove_dc_offset" : "Subtract mean from waveform on each frame (bool, default = true)" +"is_fbank" : "If true, compute power spetrum without frame energy. If false, using the frame energy instead of the square of the constant component of the signal. (bool, default = true)" +"output_type" : "If 1, return power spectrum. If 2, return log-power spectrum. (int, default = 1)" +"upper_frequency_limit" : "High cutoff frequency for mel bins (if <= 0, offset from Nyquist) (float, default = 0)" +"lower_frequency_limit" : "Low cutoff frequency for mel bins (float, default = 20)" +"filterbank_channel_count" : "Number of triangular mel-frequency bins (float, default = 23)" +"dither" : "Dithering constant (0.0 means no dither) (float, default = 1) [add robust to training]" + +#TODO +"use-energy" : "Add an extra dimension with energy to the FBANK output. (bool, default = false)" +``` + +## 4. Pitch + +This model extracts (pitch, NCCF) per frame. [Normalized Cross Correlation Function (NCCF)] + +### 4.1 pitch.py + +#### 4.1.1 Usage + +```python +from athena.transform.feats.pitch import Pitch +conf = {'window_length':0.025, 'soft_min_f0':10.0} +pitch = Pitch.params(conf).instantiate() +pitch_test = pitch(input_data, sample_rate) +``` + +#### 4.1.2 Configures Setting[Options] + +```python +"delta-pitch" : "Smallest relative change in pitch that our algorithm measures (float, default = 0.005)" +"frame-length" : "Frame length in seconds (float, default = 0.025)" +"frame-shift" : "Frame shift in seconds (float, default = 0.010)" +"frames-per-chunk" : "Only relevant for offline pitch extraction (e.g. compute-kaldi-pitch-feats), you can set it to a small nonzero value, such as 10, for better feature compatibility with online decoding (affects energy normalization in the algorithm) (int, default = 0)" +"lowpass-cutoff" : "cutoff frequency for LowPass filter (Hz) (float, default = 1000)" +"lowpass-filter-width" : "Integer that determines filter width of lowpass filter, more gives sharper filter (int, default = 1)" +"max-f0" : "max. F0 to search for (Hz) (float, default = 400)" +"max-frames-latency" : "Maximum number of frames of latency that we allow pitch tracking to introduce into the feature processing (affects output only if frames-per-chunk > 0 and simulate-first-pass-online=true (int, default = 0)" +"min-f0" : "min. F0 to search for (Hz) (float, default = 50)" +"nccf-ballast" : "Increasing this factor reduces NCCF for quiet frames (float, default = 7000)" +"nccf-ballast-online" : "This is useful mainly for debug; it affects how the NCCF ballast is computed. (bool, default = false)" +"penalty-factor" : "cost factor for FO change. (float, default = 0.1)" +"preemphasis-coefficient" : "Coefficient for use in signal preemphasis (deprecated) (float, default = 0)" +"recompute-frame" : "Only relevant for online pitch extraction, or for compatibility with online pitch extraction. A non-critical parameter; the frame at which we recompute some of the forward pointers, after revising our estimate of the signal energy. Relevant if--frames-per-chunk > 0 (int, default = 500)" +"resample-frequency" : "Frequency that we down-sample the signal to. Must be more than twice lowpass-cutoff (float, default = 4000)" +"simulate-first-pass-online" : "If true, compute-kaldi-pitch-feats will output features that correspond to what an online decoder would see in the first pass of decoding-- not the final version of the features, which is the default. Relevant if frames-per-chunk > 0 (bool, default = false)" +"snip-edges" : "If this is set to false, the incomplete frames near the ending edge won't be snipped, so that the number of frames is the file size divided by the frame-shift. This makes different types of features give the same number of frames. (bool, default = true)" +"soft-min-f0" : "Minimum f0, applied in soft way, must not exceed min-f0 (float, default = 10)" +"upsample-filter-width" : "Integer that determines filter width when upsampling NCCF (int, default = 5)" +``` + +## 5. MFCC + +This model extracts MFCC features per frame. + +### 5.1 mfcc.py + +#### 5.1.1 Usage + +```python +from athena.transform.feats.mfcc import Mfcc +config = {'use_energy':True} +mfcc = Mfcc.params(conf).instantiate() +mfcc_test = mfcc(input_data, sample_rate) +``` + +#### 5.1.2 Configures Setting [Options] + +```python +"window_length" : "Window length in seconds. (float, default = 0.025)" +"frame_length" : "Hop length in seconds. (float, default = 0.010)" +"snip_edges" : "If 1, the last frame (shorter than window_length) will be cutoff. If 2, 1 // 2 frame_length data will be padded to data. (int, default = 1)" +"-raw_energy" : "If 1, compute frame energy before preemphasis and windowing. If 2, compute frame energy after preemphasis and windowing. (int, default = 1)" +"preEph_coeff" : "Coefficient for use in frame-signal preemphasis. (float, default = 0.97)" +"window_type" : "Type of window ('hamm'|'hann'|'povey'|'rect'|'blac'|'tria'). (string, default='povey')" +"remove_dc_offset" : "Subtract mean from waveform on each frame (bool, default = true)" +"is_fbank" : "If true, compute power spetrum without frame energy. If false, using the frame energy instead of the square of the constant component of the signal. (bool, default = true)" +"output_type" : "If 1, return power spectrum. If 2, return log-power spectrum. (int, default = 1)" +"upper_frequency_limit" : "High cutoff frequency for mel bins (if < 0, offset from Nyquist) (float, default = 0)" +"lower_frequency_limit" : "Low cutoff frequency for mel bins (float, default = 20)" +"filterbank_channel_count" : "Number of triangular mel-frequency bins (float, default = 23)" +"coefficient_count" : "Number of cepstra in MFCC computation.(int, default = 13)" +"cepstral_lifter" : "Constant that controls scaling of MFCCs.(float, default = 22)" +"use_energy" : "Use energy (not C0) in MFCC computation. (bool, default = True)” +``` +## 6. Fbank_Pitch + +This model extracts Fbank && Pitch features per frame. + +### 6.1 fbank_pitch.py + +### 6.1.1 Usage + +```python +from athena.transform.feats.fbank_pitch import FbankPitch +conf = {'window_length': 0.025, 'output_type': 1, 'frame_length': 0.010, 'dither': 0.0} +fbank_pitch = FbankPitch.params(conf).instantiate() +mfcc_test = fbank_pitch(input_data, sample_rate) +``` +### 6.1.2 Configures Setting [Options] + +```python +"window_length" : Window length in seconds. (float, default = 0.025) +"frame_length" : Hop length in seconds. (float, default = 0.010) +"snip_edges" : If 1, the last frame (shorter than window_length) will be cutoff. If 2, 1 // 2 frame_length data will be padded to data. (int, default = 1) +"raw_energy" : If 1, compute frame energy before preemphasis and windowing. If 2, compute frame energy after preemphasis and windowing. (int, default = 1) +"preEph_coeff" : Coefficient for use in frame-signal preemphasis. (float, default = 0.97) +"window_type" : Type of window ("hamm"|"hann"|"povey"|"rect"|"blac"|"tria"). (string, default = "povey") +"remove_dc_offset" : Subtract mean from waveform on each frame (bool, default = true) +"is_fbank" : If true, compute power spetrum without frame energy. If false, using the frame energy instead of the square of the constant component of the signal. (bool, default = true) +"output_type" : If 1, return power spectrum. If 2, return log-power spectrum. (int, default = 1) +"upper_frequency_limit" : High cutoff frequency for mel bins (if <= 0, offset from Nyquist) (float, default = 0) +"lower_frequency_limit" : Low cutoff frequency for mel bins (float, default = 20) +"filterbank_channel_count" : Number of triangular mel-frequency bins (float, default = 23) +"dither" : Dithering constant (0.0 means no dither) (float, default = 1) [add robust to training] +"delta-pitch" : Smallest relative change in pitch that our algorithm measures (float, default = 0.005) +"frames-per-chunk" : Only relevant for offline pitch extraction (e.g. compute-kaldi-pitch-feats), you can set it to a small nonzero value, such as 10, for better feature compatibility with online decoding (affects energy normalization in the algorithm) (int, default = 0) +"lowpass-cutoff" : cutoff frequency for LowPass filter (Hz) (float, default = 1000) +"lowpass-filter-width" : Integer that determines filter width of lowpass filter, more gives sharper filter (int, default = 1) +"max-f0" : max. F0 to search for (Hz) (float, default = 400) +"max-frames-latency" : Maximum number of frames of latency that we allow pitch tracking to introduce into the feature processing (affects output only if --frames-per-chunk > 0 and --simulate-first-pass-online=true (int, default = 0) +"min-f0" : min. F0 to search for (Hz) (float, default = 50) +"nccf-ballast" : Increasing this factor reduces NCCF for quiet frames (float, default = 7000) +"nccf-ballast-online" : This is useful mainly for debug; it affects how the NCCF ballast is computed. (bool, default = false) +"penalty-factor" : cost factor for FO change. (float, default = 0.1) +"preemphasis-coefficient" : Coefficient for use in signal preemphasis (deprecated) (float, default = 0) +"recompute-frame" : Only relevant for online pitch extraction, or for compatibility with online pitch extraction. A non-critical parameter; the frame at which we recompute some of the forward pointers, after revising our estimate of the signal energy. Relevant if--frames-per-chunk > 0 (int, default = 500) +"resample-frequency" : Frequency that we down-sample the signal to. Must be more than twice lowpass-cutoff (float, default = 4000) +"simulate-first-pass-online" : If true, compute-kaldi-pitch-feats will output features that correspond to what an online decoder would see in the first pass of decoding-- not the final version of the features, which is the default. Relevant if --frames-per-chunk > 0 (bool, default = false) +"soft-min-f0" : Minimum f0, applied in soft way, must not exceed min-f0 (float, default = 10) +"upsample-filter-width" : Integer that determines filter width when upsampling NCCF (int, default = 5) +``` \ No newline at end of file diff --git a/athena/transform/__init__.py b/athena/transform/__init__.py new file mode 100644 index 00000000..20377da2 --- /dev/null +++ b/athena/transform/__init__.py @@ -0,0 +1,20 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from athena.transform import audio_featurizer + +from athena.transform.audio_featurizer import AudioFeaturizer +from athena.transform.feats.cmvn import compute_cmvn +from athena.transform.feats.read_wav import read_wav diff --git a/athena/transform/audio_featurizer.py b/athena/transform/audio_featurizer.py new file mode 100644 index 00000000..b12e42a1 --- /dev/null +++ b/athena/transform/audio_featurizer.py @@ -0,0 +1,93 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The model provides a general interface for feature extraction.""" + +import tensorflow as tf +from athena.transform import feats + + +class AudioFeaturizer: + """ + Interface of audio Features extractions. + """ + #pylint: disable=dangerous-default-value + def __init__(self, config={"type": "Fbank"}): + """init + :param name Feature name, eg fbank, mfcc, plp ... + :param config + 'type': 'ReadWav', 'Fbank', 'Spectrum' + The config for fbank + 'sample_rate' 16000 + 'window_length' 0.025 + 'frame_length' 0.010 + 'upper_frequency_limit' 20 + 'filterbank_channel_count' 40 + The config for Spectrum + 'sample_rate' 16000 + 'window_length' 0.025 + 'frame_length' 0.010 + 'output_type' 1 + """ + + assert "type" in config + + self.name = config["type"] + self.feat = getattr(feats, self.name).params(config).instantiate() + + if self.name != "ReadWav": + self.read_wav = getattr(feats, "ReadWav").params(config).instantiate() + + #pylint:disable=invalid-name + def __call__(self, audio=None, sr=None, speed=1.0): + """extract feature from audo data + :param audio data or audio file + :sr sample rate + :return feature + """ + + if audio is not None and not tf.is_tensor(audio): + audio = tf.convert_to_tensor(audio) + if sr is not None and not tf.is_tensor(sr): + sr = tf.convert_to_tensor(sr) + + return self.__impl(audio, sr, speed) + + @tf.function + def __impl(self, audio=None, sr=None, speed=1.0): + """ + :param audio data or audio file, a tensor + :sr sample rate, a tensor + :return feature + """ + if self.name == "ReadWav" or self.name == "CMVN": + return self.feat(audio, speed) + elif audio.dtype is tf.string: + audio_data, sr = self.read_wav(audio, speed) + return self.feat(audio_data, sr) + else: + return self.feat(audio, sr) + + @property + def dim(self): + """return the dimension of the feature + if only ReadWav, return 1 + """ + return self.feat.dim() + + @property + def num_channels(self): + """return the channel of the feature""" + return self.feat.num_channels() diff --git a/athena/transform/feats/__init__.py b/athena/transform/feats/__init__.py new file mode 100644 index 00000000..35c04f5a --- /dev/null +++ b/athena/transform/feats/__init__.py @@ -0,0 +1,24 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from athena.transform.feats.read_wav import ReadWav +from athena.transform.feats.spectrum import Spectrum +from athena.transform.feats.framepow import Framepow +from athena.transform.feats.pitch import Pitch +from athena.transform.feats.mfcc import Mfcc +from athena.transform.feats.write_wav import WriteWav +from athena.transform.feats.fbank import Fbank +from athena.transform.feats.cmvn import CMVN, compute_cmvn +from athena.transform.feats.fbank_pitch import FbankPitch diff --git a/athena/transform/feats/base_frontend.py b/athena/transform/feats/base_frontend.py new file mode 100644 index 00000000..cda28ceb --- /dev/null +++ b/athena/transform/feats/base_frontend.py @@ -0,0 +1,58 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" base interface of Frontend """ + +import abc +import tensorflow as tf + + +class ABCFrontend(metaclass=abc.ABCMeta): + """ abstract of Frontend """ + + def __init__(self, config): + raise NotImplementedError() + + @abc.abstractmethod + def call(self, *args): + """ implementation func """ + raise NotImplementedError() + + +class BaseFrontend(ABCFrontend): + """ wrapper of abstrcat Frontend""" + + def __init__(self, config: dict): + self._config = config + + @property + def config(self): + """ config property """ + return self._config + + @classmethod + def params(cls, config=None): + """ set params """ + raise NotImplementedError() + + def __call__(self, *args): + """ call """ + return self.call(*args) + + def dim(self): + return 1 + + def num_channels(self): + return 1 diff --git a/athena/transform/feats/cmvn.py b/athena/transform/feats/cmvn.py new file mode 100644 index 00000000..d3813a46 --- /dev/null +++ b/athena/transform/feats/cmvn.py @@ -0,0 +1,86 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""This model doed CMVN on features.""" + +import numpy as np + +import tensorflow as tf + +from athena.utils.hparam import HParams +from athena.transform.feats.base_frontend import BaseFrontend + + +class CMVN(BaseFrontend): + """ + Do CMVN on features. + """ + def __init__(self, config: dict): + super().__init__(config) + + self.global_cmvn = False + if len(config.global_mean) > 1: + self.global_cmvn = True + + @classmethod + def params(cls, config=None): + """ set params """ + + hparams = HParams(cls=cls) + hparams.add_hparam("type", "CMVN") + hparams.add_hparam("global_mean", [0.0]) + hparams.add_hparam("global_variance", [1.0]) + hparams.add_hparam("local_cmvn", False) + + if config is not None: + hparams.parse(config, True) + + assert len(hparams.global_mean) == len( + hparams.global_variance + ), "Error, global_mean length {} is not equals to global_variance length {}".format( + len(hparams.global_mean), len(hparams.global_variance) + ) + + hparams.global_variance = (np.sqrt(hparams.global_variance) + 1e-6).tolist() + return hparams + + def call(self, audio_feature, speed=1.0): + params = self.config + if self.global_cmvn: + audio_feature = ( + audio_feature - params.global_mean + ) / params.global_variance + + if params.local_cmvn: + mean, var = tf.compat.v1.nn.moments(audio_feature, axes=0) + audio_feature = (audio_feature - mean) / ( + tf.compat.v1.math.sqrt(var) + 1e-6 + ) + + return audio_feature + + def dim(self): + params = self.config + return len(params.global_mean) + + +def compute_cmvn(audio_feature, mean=None, variance=None, local_cmvn=False): + if mean is not None: + assert variance is not None + audio_feature = (audio_feature - mean) / variance + if local_cmvn: + mean, var = tf.compat.v1.nn.moments(audio_feature, axes=0) + audio_feature = (audio_feature - mean) / (tf.compat.v1.math.sqrt(var) + 1e-6) + return audio_feature diff --git a/athena/transform/feats/cmvn_test.py b/athena/transform/feats/cmvn_test.py new file mode 100644 index 00000000..2c3fbb2b --- /dev/null +++ b/athena/transform/feats/cmvn_test.py @@ -0,0 +1,52 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The model tests CMVN OP.""" + +import numpy as np +import tensorflow as tf +from athena.transform.feats.cmvn import CMVN + + +class CMVNTest(tf.test.TestCase): + """ + CMVN test. + """ + def test_cmvn(self): + dim = 40 + cmvn = CMVN.params( + {"mean": np.zeros(dim).tolist(), "variance": np.ones(dim).tolist()} + ).instantiate() + audio_feature = tf.random_uniform(shape=[3, 40], dtype=tf.float32, maxval=1.0) + print(audio_feature) + normalized = cmvn(audio_feature) + print("normalized = ", normalized) + print("dim is ", cmvn.dim()) + + cmvn = CMVN.params( + { + "mean": np.zeros(dim).tolist(), + "variance": np.ones(dim).tolist(), + "cmvn": False, + } + ).instantiate() + normalized = cmvn(audio_feature) + self.assertAllClose(audio_feature, normalized) + + +if __name__ == "__main__": + if tf.__version__ < "2.0.0": + tf.compat.v1.enable_eager_execution() + tf.test.main() diff --git a/athena/transform/feats/fbank.py b/athena/transform/feats/fbank.py new file mode 100644 index 00000000..99a3277c --- /dev/null +++ b/athena/transform/feats/fbank.py @@ -0,0 +1,157 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""This model extracts Fbank features per frame.""" + +import tensorflow as tf +from athena.utils.hparam import HParams +from athena.transform.feats.ops import py_x_ops +from athena.transform.feats.base_frontend import BaseFrontend +from athena.transform.feats.spectrum import Spectrum +from athena.transform.feats.cmvn import CMVN + + +class Fbank(BaseFrontend): + """ + Computing filter banks is applying triangular filters on a Mel-scale to the power + spectrum to extract frequency bands. Return a float tensor with shape + (num_channels, num_frames, num_frequencies). + """ + def __init__(self, config: dict): + super().__init__(config) + self.spect = Spectrum(config) + self.cmvn = CMVN(config) + + # global cmvn dim == feature dim + if config.type == "Fbank" and self.cmvn.global_cmvn: + assert config.filterbank_channel_count * config.channel == len( + config.global_mean + ), "Error, feature dim {} is not equals to cmvn dim {}".format( + config.filterbank_channel_count * config.channel, + len(config.global_mean), + ) + print("Fbank params: ", self.config) + + @classmethod + def params(cls, config=None): + """ + Set params. + :param config: contains thirteen optional parameters: + --window_length : Window length in seconds. (float, default = 0.025) + --frame_length : Hop length in seconds. (float, default = 0.010) + --snip_edges : If 1, the last frame (shorter than window_length) will be + cutoff. If 2, 1 // 2 frame_length data will be padded + to data. (int, default = 1) + ---raw_energy : If 1, compute frame energy before preemphasis and + windowing. If 2, compute frame energy after + preemphasis and windowing. (int, default = 1) + --preEph_coeff : Coefficient for use in frame-signal preemphasis. + (float, default = 0.97) + --window_type : Type of window ("hamm"|"hann"|"povey"|"rect"|"blac"|"tria"). + (string, default = "povey") + --remove_dc_offset : Subtract mean from waveform on each frame. + (bool, default = true) + --is_fbank : If true, compute power spetrum without frame energy. + If false, using the frame energy instead of the + square of the constant component of the signal. + (bool, default = true) + --output_type : If 1, return power spectrum. If 2, return log-power + spectrum. (int, default = 1) + --upper_frequency_limit : High cutoff frequency for mel bins (if <= 0, offset + from Nyquist) (float, default = 0) + --lower_frequency_limit : Low cutoff frequency for mel bins (float, default = 20) + --filterbank_channel_count : Number of triangular mel-frequency bins. + (float, default = 23) + --dither : Dithering constant (0.0 means no dither). + (float, default = 1) [add robust to training] + :return: An object of class HParams, which is a set of hyperparameters as name-value pairs. + """ + + hparams = HParams(cls=cls) + + # spectrum + hparams.append(Spectrum.params({"output_type": 1, "is_fbank": True})) + + # fbank + upper_frequency_limit = 0 + lower_frequency_limit = 60 + filterbank_channel_count = 40 + hparams.add_hparam("upper_frequency_limit", upper_frequency_limit) + hparams.add_hparam("lower_frequency_limit", lower_frequency_limit) + hparams.add_hparam("filterbank_channel_count", filterbank_channel_count) + + # delta + delta_delta = False # True + order = 2 + window = 2 + hparams.add_hparam("delta_delta", delta_delta) + hparams.add_hparam("order", order) + hparams.add_hparam("window", window) + + if config is not None: + hparams.parse(config, True) + + hparams.type = "Fbank" + + hparams.add_hparam("channel", 1) + if hparams.delta_delta: + hparams.channel = hparams.order + 1 + + return hparams + + def call(self, audio_data, sample_rate): + """ + Caculate fbank features of audio data. + :param audio_data: the audio signal from which to compute spectrum. + Should be an (1, N) tensor. + :param sample_rate: [option]the samplerate of the signal we working with, + default is 16kHz. + :return: A float tensor of size (num_channels, num_frames, num_frequencies) containing + fbank features of every frame in speech. + """ + p = self.config + + with tf.name_scope('fbank'): + + spectrum = self.spect(audio_data, sample_rate) + spectrum = tf.expand_dims(spectrum, 0) + sample_rate = tf.cast(sample_rate, dtype=tf.int32) + + fbank = py_x_ops.fbank(spectrum, + sample_rate, + upper_frequency_limit=p.upper_frequency_limit, + lower_frequency_limit=p.lower_frequency_limit, + filterbank_channel_count=p.filterbank_channel_count) + + fbank = tf.squeeze(fbank, axis=0) + shape = tf.shape(fbank) + nframe = shape[0] + nfbank = shape[1] + if p.delta_delta: + fbank = py_x_ops.delta_delta(fbank, p.order, p.window) + if p.type == 'Fbank': + fbank = self.cmvn(fbank) + + fbank = tf.reshape(fbank, (nframe, nfbank, p.channel)) + + return fbank + + def dim(self): + p = self.config + return p.filterbank_channel_count + + def num_channels(self): + p = self.config + return p.channel diff --git a/athena/transform/feats/fbank_pitch.py b/athena/transform/feats/fbank_pitch.py new file mode 100644 index 00000000..540cb503 --- /dev/null +++ b/athena/transform/feats/fbank_pitch.py @@ -0,0 +1,215 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""This model extracts Fbank && Pitch features per frame.""" + +import tensorflow as tf +from athena.utils.hparam import HParams +from athena.transform.feats.base_frontend import BaseFrontend +from athena.transform.feats.pitch import Pitch +from athena.transform.feats.fbank import Fbank +from athena.transform.feats.cmvn import CMVN + +class FbankPitch(BaseFrontend): + """ + Compute Fbank && Pitch features respectively，and concate them. Return + a tensor with shape (num_frames, dim_features). + """ + def __init__(self, config: dict): + super().__init__(config) + self.fbank = Fbank(config) + self.pitch = Pitch(config) + self.cmvn = CMVN(config) + + @classmethod + def params(cls, config=None): + """ + Set params. + :param config: contains twenty-nine optional parameters:t + --window_length : Window length in seconds. (float, default = 0.025) + --frame_length : Hop length in seconds. (float, default = 0.010) + --snip_edges : If 1, the last frame (shorter than window_length) will + be cutoff. If 2, 1 // 2 frame_length data will be padded + to data. (int, default = 1) + ---raw_energy : If 1, compute frame energy before preemphasis and + windowing. If 2, compute frame energy after preemphasis + and windowing. (int, default = 1) + --preEph_coeff : Coefficient for use in frame-signal preemphasis. + (float, default = 0.97) + --window_type : Type of window ("hamm"|"hann"|"povey"|"rect"|"blac"|"tria"). + (string, default = "povey") + --remove_dc_offset : Subtract mean from waveform on each frame. + (bool, default = true) + --is_fbank : If true, compute power spetrum without frame + energy. If false, using the frame energy instead + of the square of the constant component of the + signal. (bool, default = true) + --output_type : If 1, return power spectrum. If 2, return + log-power spectrum. (int, default = 1) + --upper_frequency_limit : High cutoff frequency for mel bins. + (if <= 0, offset from Nyquist) (float, default = 0) + --lower_frequency_limit : Low cutoff frequency for mel bins. + (float, default = 20) + --filterbank_channel_count : Number of triangular mel-frequency bins. + (float, default = 23) + --dither : Dithering constant (0.0 means no dither). + (float, default = 1) + [add robust to training] + --delta-pitch : Smallest relative change in pitch that our + algorithm measures. (float, default = 0.005) + --frames-per-chunk : Only relevant for offline pitch extraction. + (e.g. compute-kaldi-pitch-feats), you can set it to a + small nonzero value, such as 10, for better feature + compatibility with online decoding (affects energy + normalization in the algorithm) (int, default = 0) + --lowpass-cutoff : cutoff frequency for LowPass filter (Hz). + (float, default = 1000) + --lowpass-filter-width : Integer that determines filter width of lowpass filter, + more gives sharper filter (int, default = 1) + --max-f0 : max. F0 to search for (Hz) (float, default = 400) + --max-frames-latency : Maximum number of frames of latency that we allow pitch + tracking to introduce into the feature processing + (affects output only if --frames-per-chunk > 0 and + --simulate-first-pass-online=true (int, default = 0) + --min-f0 : min. F0 to search for (Hz) (float, default = 50) + --nccf-ballast : Increasing this factor reduces NCCF for quiet frames. + (float, default = 7000) + --nccf-ballast-online : This is useful mainly for debug; it affects how the + NCCF ballast is computed. (bool, default = false) + --penalty-factor : cost factor for FO change. (float, default = 0.1) + --preemphasis-coefficient : Coefficient for use in signal preemphasis (deprecated) + (float, default = 0) + --recompute-frame : Only relevant for online pitch extraction, or for + compatibility with online pitch extraction. A + non-critical parameter; the frame at which we recompute + some of the forward pointers, after revising our + estimate of the signal energy. Relevant + if--frames-per-chunk > 0. (int, default = 500) + --resample-frequency : Frequency that we down-sample the signal to. Must be + more than twice lowpass-cutoff (float, default = 4000) + --simulate-first-pass-online : If true, compute-kaldi-pitch-feats will output features + that correspond to what an online decoder would see in + the first pass of decoding-- not the final version of + the features, which is the default. Relevant if + --frames-per-chunk > 0 (bool, default = false) + --soft-min-f0 : Minimum f0, applied in soft way, must not exceed + min-f0 (float, default = 10) + --upsample-filter-width : Integer that determines filter width when upsampling + NCCF (int, default = 5) + :return: An object of class HParams, which is a set of hyperparameters as name-value pairs. + """ + hparams = HParams(cls=cls) + hparams.append(CMVN.params()) + + upper_frequency_limit = 0 + lower_frequency_limit = 20.0 + filterbank_channel_count = 80.0 + window_length = 0.025 + frame_length = 0.010 + raw_energy = 1 + preEph_coeff = 0.97 + window_type = 'povey' + remove_dc_offset = True + is_fbank = True + output_type = 1 + dither = 0.0 + snip_edges = True + preemph_coeff = 0.0 + min_f0 = 50.0 + max_f0 = 400.0 + soft_min_f0 = 10.0 + penalty_factor = 0.1 + lowpass_cutoff = 1000.0 + resample_freq = 4000.0 + delta_pitch = 0.005 + nccf_ballast = 7000.0 + lowpass_filter_width = 1 + upsample_filter_width = 5 + max_frames_latency = 0 + frames_per_chunk = 0 + simulate_first_pass_online = False + recompute_frame = 500 + nccf_ballast_online = False + + # delta + delta_delta = False # True + order = 2 + window = 2 + hparams.add_hparam('delta_delta', delta_delta) + hparams.add_hparam('order', order) + hparams.add_hparam('window', window) + hparams.add_hparam('channel', 1) + + if hparams.delta_delta: + hparams.channel = hparams.order + 1 + + hparams.add_hparam('snip_edges', snip_edges) + hparams.add_hparam('preemph_coeff', preemph_coeff) + hparams.add_hparam('min_f0', min_f0) + hparams.add_hparam('max_f0', max_f0) + hparams.add_hparam('dither', dither) + hparams.add_hparam('soft_min_f0', soft_min_f0) + hparams.add_hparam('penalty_factor', penalty_factor) + hparams.add_hparam('lowpass_cutoff', lowpass_cutoff) + hparams.add_hparam('resample_freq', resample_freq) + hparams.add_hparam('delta_pitch', delta_pitch) + hparams.add_hparam('nccf_ballast', nccf_ballast) + hparams.add_hparam('lowpass_filter_width', lowpass_filter_width) + hparams.add_hparam('upsample_filter_width', upsample_filter_width) + hparams.add_hparam('max_frames_latency', max_frames_latency) + hparams.add_hparam('frames_per_chunk', frames_per_chunk) + hparams.add_hparam('simulate_first_pass_online', + simulate_first_pass_online) + hparams.add_hparam('recompute_frame', recompute_frame) + hparams.add_hparam('nccf_ballast_online', nccf_ballast_online) + hparams.add_hparam('upper_frequency_limit', upper_frequency_limit) + hparams.add_hparam('lower_frequency_limit', lower_frequency_limit) + hparams.add_hparam('filterbank_channel_count', filterbank_channel_count) + hparams.add_hparam('window_length', window_length) + hparams.add_hparam('frame_length', frame_length) + hparams.add_hparam('output_type', output_type) + hparams.add_hparam('raw_energy', raw_energy) + hparams.add_hparam('preEph_coeff', preEph_coeff) + hparams.add_hparam('window_type', window_type) + hparams.add_hparam('remove_dc_offset', remove_dc_offset) + hparams.add_hparam('is_fbank', is_fbank) + + if config is not None: + hparams.parse(config, True) + + return hparams + + def call(self, audio_data, sample_rate): + """ + Caculate fbank && pitch(concat) features of wav. + :param audio_data: the audio signal from which to compute spectrum. + Should be an (1, N) tensor. + :param sample_rate: the samplerate of the signal we working with. + :return: A tensor with shape (num_frames, dim_features), containing + fbank && pitch feature of every frame in speech. + """ + + with tf.name_scope('fbank_pitch'): + + fbank_feats = tf.squeeze(self.fbank(audio_data, sample_rate)) + pitch_feats = tf.squeeze(self.pitch(audio_data, sample_rate)) + fbank_pitch_feats = tf.concat([fbank_feats, pitch_feats], 1) + fbank_pitch_feats = tf.expand_dims(fbank_pitch_feats, 2) + + return fbank_pitch_feats + + def dim(self): + res = self.fbank.dim() + self.pitch.dim() + return int(res) \ No newline at end of file diff --git a/athena/transform/feats/fbank_pitch_test.py b/athena/transform/feats/fbank_pitch_test.py new file mode 100644 index 00000000..ad6cc34c --- /dev/null +++ b/athena/transform/feats/fbank_pitch_test.py @@ -0,0 +1,56 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The model tests Fbank&&Pitch FE.""" + +import os +from pathlib import Path +import tensorflow as tf +from tensorflow.python.framework.ops import disable_eager_execution +from athena.transform.feats.read_wav import ReadWav +from athena.transform.feats.fbank_pitch import FbankPitch + +os.environ['CUDA_VISIBLE_DEVICES'] = '-1' + +class FbankPitchTest(tf.test.TestCase): + """ + Fbank && Pitch extraction test. + """ + def test_FbankPitch(self): + wav_path = str(Path(os.environ['MAIN_ROOT']).joinpath('examples/sm1_cln.wav')) + + with self.session(): + read_wav = ReadWav.params().instantiate() + input_data, sample_rate = read_wav(wav_path) + config = {'window_length': 0.025, 'output_type': 1, 'frame_length': 0.010, 'dither': 0.0} + fbank_pitch = FbankPitch.params(config).instantiate() + fbank_pitch_test = fbank_pitch(input_data, sample_rate) + + if tf.executing_eagerly(): + self.assertEqual(tf.rank(fbank_pitch_test).numpy(), 3) + print(fbank_pitch_test.numpy()[0:2, :, 0]) + else: + self.assertEqual(tf.rank(fbank_pitch_test).eval(), 3) + print(fbank_pitch_test.eval()[0:2, :, 0]) + +if __name__ == '__main__': + + is_eager = True + if not is_eager: + disable_eager_execution() + else: + if tf.__version__ < '2.0.0': + tf.compat.v1.enable_eager_execution() + tf.test.main() diff --git a/athena/transform/feats/fbank_test.py b/athena/transform/feats/fbank_test.py new file mode 100644 index 00000000..b6c0e828 --- /dev/null +++ b/athena/transform/feats/fbank_test.py @@ -0,0 +1,102 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import os +from pathlib import Path +import numpy as np +import tensorflow as tf +from tensorflow.python.framework.ops import disable_eager_execution +from athena.transform.feats.read_wav import ReadWav +from athena.transform.feats.fbank import Fbank + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + + +class FbankTest(tf.test.TestCase): + def test_fbank(self): + # 16kHz && 8kHz test + wav_path_16k = str( + Path(os.environ["MAIN_ROOT"]).joinpath("examples/sm1_cln.wav") + ) + wav_path_8k = str( + Path(os.environ["MAIN_ROOT"]).joinpath("examples/english.wav") + ) + + with self.session(): + # value test + read_wav = ReadWav.params().instantiate() + input_data, sample_rate = read_wav(wav_path_16k) + fbank = Fbank.params({"delta_delta": False}).instantiate() + fbank_test = fbank(input_data, sample_rate) + real_fank_feats = np.array( + [ + [3.768338, 4.946218, 6.289874, 6.330853, 6.761764, 6.884573], + [3.803553, 5.450971, 6.547878, 5.796172, 6.397846, 7.242926], + ] + ) + # self.assertAllClose(np.squeeze(fbank_test.eval()[0:2, 0:6, 0]), + # real_fank_feats, rtol=1e-05, atol=1e-05) + if tf.executing_eagerly(): + print(fbank_test.numpy()[0:2, 0:6, 0]) + else: + print(fbank_test.eval()[0:2, 0:6, 0]) + count = 1 + + for wav_file in [wav_path_8k, wav_path_16k]: + + read_wav = ReadWav.params().instantiate() + input_data, sample_rate = read_wav(wav_file) + if tf.executing_eagerly(): + print(wav_file, sample_rate.numpy()) + else: + print(wav_file, sample_rate.eval()) + + conf = { + "delta_delta": True, + "lower_frequency_limit": 100, + "upper_frequency_limit": 0, + } + fbank = Fbank.params(conf).instantiate() + fbank_test = fbank(input_data, sample_rate) + if tf.executing_eagerly(): + print(fbank_test.numpy()) + else: + print(fbank_test.eval()) + print(fbank.num_channels()) + + conf = { + "delta_delta": False, + "lower_frequency_limit": 100, + "upper_frequency_limit": 0, + } + fbank = Fbank.params(conf).instantiate() + fbank_test = fbank(input_data, sample_rate) + print(fbank_test) + print(fbank.num_channels()) + count += 1 + del read_wav + del fbank + + +if __name__ == "__main__": + + is_eager = True + if not is_eager: + disable_eager_execution() + else: + if tf.__version__ < "2.0.0": + tf.compat.v1.enable_eager_execution() + tf.test.main() diff --git a/athena/transform/feats/framepow.py b/athena/transform/feats/framepow.py new file mode 100644 index 00000000..75088c82 --- /dev/null +++ b/athena/transform/feats/framepow.py @@ -0,0 +1,87 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""""This model extracts framepow features per frame.""" + +import tensorflow as tf +from athena.utils.hparam import HParams +from athena.transform.feats.ops import py_x_ops +from athena.transform.feats.base_frontend import BaseFrontend + + +class Framepow(BaseFrontend): + """ + Compute power of every frame in speech. Return a float tensor with + shape (1 * num_frames). + """ + def __init__(self, config: dict): + super().__init__(config) + + @classmethod + def params(cls, config=None): + """ + Set params. + :param config: contains four optional parameters: + --window_length : Window length in seconds. (float, default = 0.025) + --frame_length : Hop length in seconds. (float, default = 0.010) + --snip_edges : If True, the last frame (shorter than window_length) + will be cutoff. If False, 1 // 2 frame_length data will + be padded to data. (int, default = True) + --remove_dc_offset : Subtract mean from waveform on each frame (bool, default = true) + :return:An object of class HParams, which is a set of hyperparameters as name-value pairs. + """ + + window_length = 0.025 + frame_length = 0.010 + snip_edges = 1 + remove_dc_offset = True + + hparams = HParams(cls=cls) + hparams.add_hparam("window_length", window_length) + hparams.add_hparam("frame_length", frame_length) + hparams.add_hparam("snip_edges", snip_edges) + hparams.add_hparam("remove_dc_offset", remove_dc_offset) + + if config is not None: + hparams.parse(config, True) + + return hparams + + def call(self, audio_data, sample_rate): + """ + Caculate power of every frame in speech. + :param audio_data: the audio signal from which to compute spectrum. + Should be an (1, N) tensor. + :param sample_rate: [option]the samplerate of the signal we working with, + default is 16kHz. + :return:A float tensor of size (1 * num_frames) containing power of every + frame in speech. + """ + + p = self.config + with tf.name_scope('framepow'): + sample_rate = tf.cast(sample_rate, dtype=float) + framepow = py_x_ops.frame_pow( + audio_data, + sample_rate, + snip_edges=p.snip_edges, + remove_dc_offset=p.remove_dc_offset, + window_length=p.window_length, + frame_length=p.frame_length) + + return tf.squeeze(framepow) + + def dim(self): + return 1 \ No newline at end of file diff --git a/athena/transform/feats/framepow_test.py b/athena/transform/feats/framepow_test.py new file mode 100644 index 00000000..7b92c137 --- /dev/null +++ b/athena/transform/feats/framepow_test.py @@ -0,0 +1,74 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The model tests framepow FE.""" + +import os +from pathlib import Path +import numpy as np +import tensorflow as tf +from tensorflow.python.framework.ops import disable_eager_execution +from athena.transform.feats.read_wav import ReadWav +from athena.transform.feats.framepow import Framepow + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + + +class FramePowTest(tf.test.TestCase): + """ + Framepow extraction test. + """ + def test_framepow(self): + wav_path_16k = str( + Path(os.environ["MAIN_ROOT"]).joinpath("examples/sm1_cln.wav") + ) + + with self.session(): + read_wav = ReadWav.params().instantiate() + input_data, sample_rate = read_wav(wav_path_16k) + config = {"snip_edges": 1} + framepow = Framepow.params(config).instantiate() + framepow_test = framepow(input_data, sample_rate) + + real_framepow_feats = np.array( + [9.819611, 9.328745, 9.247337, 9.26451, 9.266059] + ) + + if tf.executing_eagerly(): + self.assertAllClose( + framepow_test.numpy()[0:5], + real_framepow_feats, + rtol=1e-05, + atol=1e-05, + ) + print(framepow_test.numpy()[0:5]) + else: + self.assertAllClose( + framepow_test.eval()[0:5], + real_framepow_feats, + rtol=1e-05, + atol=1e-05, + ) + print(framepow_test.eval()[0:5]) + + +if __name__ == "__main__": + is_eager = True + if not is_eager: + disable_eager_execution() + else: + if tf.__version__ < "2.0.0": + tf.compat.v1.enable_eager_execution() + tf.test.main() diff --git a/athena/transform/feats/mfcc.py b/athena/transform/feats/mfcc.py new file mode 100644 index 00000000..a96286d7 --- /dev/null +++ b/athena/transform/feats/mfcc.py @@ -0,0 +1,153 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""This model extracts MFCC features per frame.""" + +import tensorflow as tf +from athena.utils.hparam import HParams +from athena.transform.feats.ops import py_x_ops +from athena.transform.feats.base_frontend import BaseFrontend +from athena.transform.feats.framepow import Framepow +from athena.transform.feats.fbank import Fbank +from athena.transform.feats.cmvn import CMVN + + +class Mfcc(BaseFrontend): + """ + Compute mfcc features of every frame in speech, return a float tensor + with size (num_channels, num_frames, num_frequencies). + """ + def __init__(self, config: dict): + super().__init__(config) + self.framepow = Framepow(config) + self.fbank = Fbank(config) + + @classmethod + def params(cls, config=None): + """ + Set params. + :param config: contains fourteen optional parameters. + --window_length : Window length in seconds. (float, default = 0.025) + --frame_length : Hop length in seconds. (float, default = 0.010) + --snip_edges : If 1, the last frame (shorter than window_length) will + be cutoff. If 2, 1 // 2 frame_length data will be padded + to data. (int, default = 1) + ---raw_energy : If 1, compute frame energy before preemphasis and + windowing. If 2, compute frame energy after + preemphasis and windowing. (int, default = 1) + --preEph_coeff : Coefficient for use in frame-signal preemphasis. + (float, default = 0.97) + --window_type : Type of window ("hamm"|"hann"|"povey"|"rect"|"blac"|"tria"). + (string, default = "povey") + --remove_dc_offset : Subtract mean from waveform on each frame + (bool, default = true) + --is_fbank : If true, compute power spetrum without frame energy. If + false, using the frame energy instead of the square of the + constant component of the signal. (bool, default = true) + --output_type : If 1, return power spectrum. If 2, return log-power + spectrum. (int, default = 1) + --upper_frequency_limit : High cutoff frequency for mel bins (if < 0, offset from + Nyquist) (float, default = 0) + --lower_frequency_limit : Low cutoff frequency for mel bins (float, default = 20) + --filterbank_channel_count : Number of triangular mel-frequency bins. + (float, default = 23) + --coefficient_count : Number of cepstra in MFCC computation. + (int, default = 13) + --cepstral_lifter : Constant that controls scaling of MFCCs. + (float, default = 22) + --use_energy :Use energy (not C0) in MFCC computation. + (bool, default = True) + :return: An object of class HParams, which is a set of hyperparameters as name-value pairs. + """ + + upper_frequency_limit = 0.0 + lower_frequency_limit = 20.0 + filterbank_channel_count = 23.0 + window_length = 0.025 + frame_length = 0.010 + output_type = 1 + snip_edges = 1 + raw_energy = 1 + preEph_coeff = 0.97 + window_type = "povey" + remove_dc_offset = True + is_fbank = True + cepstral_lifter = 22.0 + coefficient_count = 13 + use_energy = True + dither = 0.0 + delta_delta = False + order = 2 + window = 2 + + hparams = HParams(cls=cls) + hparams.add_hparam("upper_frequency_limit", upper_frequency_limit) + hparams.add_hparam("lower_frequency_limit", lower_frequency_limit) + hparams.add_hparam("filterbank_channel_count", filterbank_channel_count) + hparams.add_hparam("window_length", window_length) + hparams.add_hparam("frame_length", frame_length) + hparams.add_hparam("output_type", output_type) + hparams.add_hparam("snip_edges", snip_edges) + hparams.add_hparam("raw_energy", raw_energy) + hparams.add_hparam("preEph_coeff", preEph_coeff) + hparams.add_hparam("window_type", window_type) + hparams.add_hparam("remove_dc_offset", remove_dc_offset) + hparams.add_hparam("is_fbank", is_fbank) + hparams.add_hparam("cepstral_lifter", cepstral_lifter) + hparams.add_hparam("coefficient_count", coefficient_count) + hparams.add_hparam("use_energy", use_energy) + hparams.add_hparam("dither", dither) + hparams.add_hparam("delta_delta", delta_delta) + hparams.add_hparam("order", order) + hparams.add_hparam("window", window) + hparams.add_hparam("channel", 1) + + hparams.append(CMVN.params()) + + if config is not None: + hparams.parse(config, True) + + return hparams + + def call(self, audio_data, sample_rate): + """ + Caculate mfcc features of audio data. + :param audio_data: the audio signal from which to compute spectrum. + Should be an (1, N) tensor. + :param sample_rate: the samplerate of the signal we working with. + :return: A float tensor of size (num_channels, num_frames, num_frequencies) + containing mfcc features of every frame in speech. + """ + p = self.config + with tf.name_scope('mfcc'): + fbank_feats = self.fbank(audio_data, sample_rate) + sample_rate = tf.cast(sample_rate, dtype=tf.int32) + shape = tf.shape(fbank_feats) + nframe = shape[0] + nfbank = shape[1] + fbank_feats = tf.reshape(fbank_feats, (p.channel, nframe, nfbank)) + framepow_feats = self.framepow(audio_data, sample_rate) + mfcc = py_x_ops.mfcc( + fbank_feats, + framepow_feats, + sample_rate, + use_energy=p.use_energy, + cepstral_lifter=p.cepstral_lifter, + coefficient_count=p.coefficient_count) + return mfcc + + def dim(self): + p = self.config + return int(p.coefficient_count) \ No newline at end of file diff --git a/athena/transform/feats/mfcc_test.py b/athena/transform/feats/mfcc_test.py new file mode 100644 index 00000000..fcc599ce --- /dev/null +++ b/athena/transform/feats/mfcc_test.py @@ -0,0 +1,76 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The model tests MFCC FE.""" + +import os +from pathlib import Path +import numpy as np +import tensorflow as tf +from tensorflow.python.framework.ops import disable_eager_execution +from athena.transform.feats.read_wav import ReadWav +from athena.transform.feats.mfcc import Mfcc + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + + +class MfccTest(tf.test.TestCase): + """ + MFCC extraction test. + """ + def test_mfcc(self): + wav_path_16k = str( + Path(os.environ["MAIN_ROOT"]).joinpath("examples/sm1_cln.wav") + ) + + with self.session(): + read_wav = ReadWav.params().instantiate() + input_data, sample_rate = read_wav(wav_path_16k) + config = {"use_energy": True} + mfcc = Mfcc.params(config).instantiate() + mfcc_test = mfcc(input_data, sample_rate) + + real_mfcc_feats = np.array( + [ + [9.819611, -30.58736, -7.088838, -10.67966, -1.646479, -4.36086], + [9.328745, -30.73371, -6.128432, -7.930599, 3.208357, -1.086456], + ] + ) + + if tf.executing_eagerly(): + self.assertAllClose( + mfcc_test.numpy()[0, 0:2, 0:6], + real_mfcc_feats, + rtol=1e-05, + atol=1e-05, + ) + else: + self.assertAllClose( + mfcc_test.eval()[0, 0:2, 0:6], + real_mfcc_feats, + rtol=1e-05, + atol=1e-05, + ) + + +if __name__ == "__main__": + + is_eager = True + if not is_eager: + disable_eager_execution() + else: + if tf.__version__ < "2.0.0": + tf.compat.v1.enable_eager_execution() + tf.test.main() diff --git a/athena/transform/feats/ops/Makefile b/athena/transform/feats/ops/Makefile new file mode 100644 index 00000000..e332953e --- /dev/null +++ b/athena/transform/feats/ops/Makefile @@ -0,0 +1,95 @@ +# Find where we're running from, so we can store generated files here. + +ifeq ($(origin MAKEFILE_DIR), undefined) +MAKEFILE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +MAIN_ROOT := $(realpath $(MAKEFILE_DIR)/../../) +endif + +#$(info $(MAKEFILE_DIR)) +#$(info $(MAIN_ROOT)) + +CXX := g++ +NVCC := nvcc +PYTHON_BIN_PATH= python3 +CC := +AR := +CXXFLAGS := +LDFLAGS := +STDLIB := + +# Try to figure out the host system +HOST_OS := +ifeq ($(OS),Windows_NT) + HOST_OS = windows +else + UNAME_S := $(shell uname -s) + ifeq ($(UNAME_S),Linux) + HOST_OS := linux + endif + ifeq ($(UNAME_S),Darwin) + HOST_OS := ios + endif +endif + +#HOST_ARCH := $(shell if [[ $(shell uname -m) =~ i[345678]86 ]]; then echo x86_32; else echo $(shell uname -m); fi) +HOST_ARCH=x86_64 +TARGET := $(HOST_OS) +TARGET_ARCH := $(HOST_ARCH) + +GENDIR := $(MAKEFILE_DIR)/gen/ +TGTDIR := $(GENDIR)$(TARGET)_$(TARGET_ARCH)/ +OBJDIR := $(TGTDIR)obj/ +BINDIR := $(TGTDIR)bin/ +LIBDIR := $(TGTDIR)lib/ + + +TF_CFLAGS := $(shell $(PYTHON_BIN_PATH) -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') +# Fix TF LDFLAGS issue on macOS. +TF_LFLAGS := $(shell $(PYTHON_BIN_PATH) -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))' | sed "s/-l:libtensorflow_framework.1.dylib/-ltensorflow_framework.1/") +#TF_INCLUDES := $(shell $(PYTHON_BIN_PATH) -c 'import tensorflow as tf; print(tf.sysconfig.get_include())') +TF_LIBS := $(shell $(PYTHON_BIN_PATH) -c 'import tensorflow as tf; print(tf.sysconfig.get_lib())') +CXXFLAGS += -fPIC -shared -O2 -std=c++11 -DFEATURE_VERSION=\"$(shell git rev-parse --short HEAD)\" $(TF_CFLAGS) +INCLUDES := -I$(MAIN_ROOT) \ + -I$(MAIN_ROOT)/feats/ops \ + +LDFLAGS += $(TF_LFLAGS) + +CORE_CC_EXCLUDE_SRCS := \ +$(wildcard kernels/*test.cc) \ +$(wildcard kernels/*test_util.cc) + +# src and tgts +LIB_SRCS_ALL := $(wildcard kernels/*.cc) +LIB_SRCS := $(filter-out $(CORE_CC_EXCLUDE_SRCS), $(LIB_SRCS_ALL)) +LIB_OBJS := $(addprefix $(OBJDIR), $(patsubst %.cc, %.o, $(patsubst %.c, %.o, $(LIB_SRCS)))) + +# lib +SHARED_LIB := x_ops.so + +TEST_SRC := $(wildcard kernels/*_test.cc) +TEST_OBJ := $(addprefix $(OBJDIR), $(patsubst %.cc, $(OBJS_DIR)%.o, $(TEST_SRC))) +TEST_BIN := $(addprefix $(BINDIR), $(patsubst %.cc, $(OBJS_DIR)%.bin, $(TEST_SRC))) +#TEST_BIN := $(BINDIR)test + +all: $(SHARED_LIB) $(TEST_BIN) + +$(OBJDIR)%.o: %.cc + @mkdir -p $(dir $@) + $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ $(LDFLAGS) + +$(SHARED_LIB): $(LIB_OBJS) + @mkdir -p $(dir $@) + $(CXX) -fPIC -shared -o $@ $^ $(STDLIB) $(LDFLAGS) + +$(STATIC_LIB): $(LIB_OBJS) + @mkdir -p $(dir $@) + $(AR) crsv $@ $^ + +${TEST_BIN}: $(TEST_OBJ) $(STATIC_LIB) + @mkdir -p $(dir $@) + $(CXX) $(LDFLAGS) $^ -o $@ $(STATIC_LIB) + +.PHONY: clean +clean: + -rm -rf $(GENDIR) + -rm -f x_ops.so diff --git a/athena/transform/feats/ops/__init__.py b/athena/transform/feats/ops/__init__.py new file mode 100644 index 00000000..11e28dcc --- /dev/null +++ b/athena/transform/feats/ops/__init__.py @@ -0,0 +1,15 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== diff --git a/athena/transform/feats/ops/kernels/complex_defines.h b/athena/transform/feats/ops/kernels/complex_defines.h new file mode 100644 index 00000000..4e16add1 --- /dev/null +++ b/athena/transform/feats/ops/kernels/complex_defines.h @@ -0,0 +1,250 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef DELTA_LAYERS_OPS_KERNELS_COMPLEX_DEFINES_H_ +#define DELTA_LAYERS_OPS_KERNELS_COMPLEX_DEFINES_H_ + +#include + +// define complex declare type +typedef struct FCOMPLEX { + float r, i; +} fcomplex; +typedef struct ICOMPLEX { + int r, i; +} icomplex; +typedef struct DCOMPLEX { + double r, i; +} dcomplex; + +// define fcomplex, float type +typedef fcomplex xcomplex; +typedef float xt; + +// complex generation +inline xcomplex complex_gen(xt re, xt im) { + xcomplex c; + + c.r = re; + c.i = im; + + return c; +} + +// complex conjugation +inline xcomplex complex_conjg(xcomplex z) { + xcomplex c; + + c.r = z.r; + c.i = -z.i; + + return c; +} + +// complex abs +inline xt complex_abs(xcomplex z) { + xt x, y, ans, temp; + + x = (xt)fabs(z.r); + y = (xt)fabs(z.i); + + if (x == 0.0) { + ans = y; + } else if (y == 0.0) { + ans = x; + } else if (x > y) { + temp = y / x; + ans = x * (xt)sqrt(1.0 + temp * temp); + } else { + temp = x / y; + ans = y * (xt)sqrt(1.0 + temp * temp); + } + + return ans; +} + +// complex number absolute value square +inline xt complex_abs2(xcomplex cp) { + return (cp.r * cp.r + cp.i * cp.i); +} + +// complex sqrt +inline xcomplex complex_sqrt(xcomplex z) { + xcomplex c; + xt x, y, w, r; + + if ((z.r == 0.0) && (z.i == 0.0)) { + c.r = 0.0; + c.i = 0.0; + + return c; + } else { + x = (xt)fabs(z.r); + y = (xt)fabs(z.i); + + if (x >= y) { + r = y / x; + w = (xt)(sqrt(x) * sqrt(0.5 * (1.0 + sqrt(1.0 + r * r)))); + } else { + r = x / y; + w = (xt)(sqrt(y) * sqrt(0.5 * (r + sqrt(1.0 + r * r)))); + } + + if (z.r >= 0.0) { + c.r = w; + c.i = z.i / (2.0f * w); + } else { + c.i = (z.i >= 0) ? w : -w; + c.r = z.i / (2.0f * c.i); + } + + return c; + } +} + +// complex addition +inline xcomplex complex_add(xcomplex a, xcomplex b) { + xcomplex c; + + c.r = a.r + b.r; + c.i = a.i + b.i; + + return c; +} + +// complex subtraction +inline xcomplex complex_sub(xcomplex a, xcomplex b) { + xcomplex c; + + c.r = a.r - b.r; + c.i = a.i - b.i; + + return c; +} + +// complex multiplication +inline xcomplex complex_mul(xcomplex a, xcomplex b) { + xcomplex c; + + c.r = a.r * b.r - a.i * b.i; + c.i = a.i * b.r + a.r * b.i; + + return c; +} + +// real and complex mutiplication +inline xcomplex complex_real_complex_mul(xt x, xcomplex a) { + xcomplex c; + + c.r = x * a.r; + c.i = x * a.i; + + return c; +} + +// complex division +inline xcomplex complex_div(xcomplex a, xcomplex b) { + xcomplex c; + xt r, den; + + if (fabs(b.r) >= fabs(b.i)) { + r = b.i / b.r; + den = b.r + r * b.i; + c.r = (a.r + r * a.i) / den; + c.i = (a.i - r * a.r) / den; + } else { + r = b.r / b.i; + den = b.i + r * b.r; + c.r = (a.r * r + a.i) / den; + c.i = (a.i * r - a.r) / den; + } + + return c; +} + +// complex division 2 +inline xcomplex complex_div2(xcomplex a, xcomplex b) { + xcomplex c; + xt den; + + den = b.r * b.r + b.i * b.i; + + c.r = (a.r * b.r + a.i * b.i) / den; + c.i = (a.i * b.r - a.r * b.i) / den; + + return c; +} + +// complex number div real number +inline xcomplex complex_div_real(xcomplex cp, xt r) { + xcomplex tmpcp; + + tmpcp.r = cp.r / r; + tmpcp.i = cp.i / r; + + return (tmpcp); +} + +// complex number averaging +inline xcomplex complex_avg_vec(xcomplex *cpVec, int cpVecLen) { + int j; + xcomplex tmpcp; + + tmpcp.r = 0.0; + tmpcp.i = 0.0; + + for (j = 0; j < cpVecLen; j++) tmpcp = complex_add(tmpcp, cpVec[j]); + + tmpcp = complex_div_real(tmpcp, (xt)cpVecLen); + + return (tmpcp); +} + +/* function implementations */ +/*--------------------------------------------------- +complex FIR filtering of 2nd dimension data +len : Tap length : in : int +hat : impulse response : in : xcomplex[][] +buf : input buffer : in : xcomplex[][] +xout: output data : out : xcomplex[] +---------------------------------------------------*/ +inline xcomplex complex_conv(int len, xcomplex *hat, xcomplex *buf) { + int i; + xcomplex z, xout; + + xout.r = xout.i = 0.0; + for (i = 0; i < len; i++) { + z = complex_mul(complex_conjg(hat[i]), buf[i]); + xout = complex_add(xout, z); + } + return (xout); +} + +/*--------------------------------------------------- +CmplxDataPush +: push complex data into 2nd dimension bufer +in len bufer length +xin input data +renewal buf bufer +---------------------------------------------------*/ +inline void complex_data_push(int len, xcomplex xin, xcomplex *buf) { + int i; + + for (i = len - 1; i >= 1; i--) buf[i] = buf[i - 1]; + buf[0] = xin; +} + +#endif // DELTA_LAYERS_OPS_KERNELS_COMPLEX_DEFINES_H_ diff --git a/athena/transform/feats/ops/kernels/delta_delta.cc b/athena/transform/feats/ops/kernels/delta_delta.cc new file mode 100644 index 00000000..65e1f0db --- /dev/null +++ b/athena/transform/feats/ops/kernels/delta_delta.cc @@ -0,0 +1,121 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "kernels/delta_delta.h" + +#include + +namespace delta { + +const int kOrder = 2; +const int kWindow = 2; + +DeltaDelta::DeltaDelta() + : initialized_(false), order_(kOrder), window_(kWindow) {} + +DeltaDelta::~DeltaDelta() { + for (int i = 0; i < scales_.size(); i++) + std::vector().swap(scales_[i]); + (scales_).clear(); +} + +bool DeltaDelta::Initialize(int order, int window) { + if (order < 0 || order > 1000) { + LOG(ERROR) << "order must be greater than zero and less than 1000."; + return false; + } + if (window < 0 || window > 1000) { + LOG(ERROR) << "window must be greater than zero and less than 1000."; + return false; + } + order_ = order; + window_ = window; + + scales_.resize(order_ + 1); + scales_[0].resize(1); + scales_[0][0] = + 1.0; // trival window for 0th order delta [i.e. baseline feats] + + for (int i = 1; i <= order_; i++) { + std::vector&prev_scales = scales_[i - 1], &cur_scales = scales_[i]; + + int window = window_; + if (window == 0) { + LOG(ERROR) << "window must not be zero."; + return false; + } + int prev_offset = (static_cast(prev_scales.size() - 1)) / 2, + cur_offset = prev_offset + window; + cur_scales.resize(prev_scales.size() + 2 * window); + + double normalizer = 0.0; + for (int j = -window; j <= window; j++) { + normalizer += j * j; + for (int k = -prev_offset; k <= prev_offset; k++) { + cur_scales[j + k + cur_offset] += + static_cast(j) * prev_scales[k + prev_offset]; + } + } + + for (int i = 0; i < cur_scales.size(); i++) { + cur_scales[i] *= (1.0 / normalizer); + } + } + + initialized_ = true; + return initialized_; +} + +// process one frame per time +void DeltaDelta::Compute(const Tensor& input_feats, int frame, + std::vector* output) const { + if (!initialized_) { + LOG(ERROR) << "DeltaDelta not initialized."; + return; + } + + int num_frames = input_feats.dim_size(0); + int feat_dim = input_feats.dim_size(1); + int output_dim = feat_dim * (order_ + 1); + + output->resize(output_dim); + auto input = input_feats.matrix(); + + for (int i = 0; i <= order_; i++) { + const std::vector& scales = scales_[i]; + int max_offset = (scales.size() - 1) / 2; + // e.g. max_offset=2, (-2, 2) + for (int j = -max_offset; j <= max_offset; j++) { + // if asked to read + int offset_frame = frame + j; + if (offset_frame < 0) + offset_frame = 0; + else if (offset_frame >= num_frames) + offset_frame = num_frames - 1; + + // sacles[0] for `-max_offset` frame + double scale = scales[j + max_offset]; + if (scale != 0.0) { + for (int k = 0; k < feat_dim; k++) { + (*output)[i + k * (order_ + 1)] += input(offset_frame, k) * scale; + } + } + } + } + return; +} + +} // namespace delta diff --git a/athena/transform/feats/ops/kernels/delta_delta.h b/athena/transform/feats/ops/kernels/delta_delta.h new file mode 100644 index 00000000..ef63ea60 --- /dev/null +++ b/athena/transform/feats/ops/kernels/delta_delta.h @@ -0,0 +1,75 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef DELTA_LAYERS_OPS_KERNELS_DELTA_DELTA_H_ +#define DELTA_LAYERS_OPS_KERNELS_DELTA_DELTA_H_ + +#include + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/logging.h" + +using namespace tensorflow; // NOLINT + +namespace delta { + +// This class provides a low-level function to compute delta features. +// The function takes as input a matrix of features and a frame index +// that it should compute the deltas on. It puts its output in an object +// of type VectorBase, of size (original-feature-dimension) * (opts.order+1). +// This is not the most efficient way to do the computation, but it's +// state-free and thus easier to understand +class DeltaDelta { + public: + DeltaDelta(); + ~DeltaDelta(); + + bool Initialize(int order, int window); + + // Input is a single feature frame. Output is populated with + // the feature, delta, delta-delta values. + void Compute(const Tensor& input_feats, int frame, + std::vector* output) const; + + void set_order(int order) { + CHECK(!initialized_) << "Set order before calling Initialize."; + order_ = order; + } + + void set_window(int window) { + CHECK(!initialized_) << "Set window before calling Initialize."; + window_ = window; + } + + private: + int order_; + // e.g. 2; controls window size (window size is 2*window + 1) + // the behavior at the edges is to replicate the first or last frame. + // this is not configurable. + int window_; + + // a scaling window for each of the orders, including zero: + // multiply the features for each dimension by this window. + std::vector > scales_; + + bool initialized_; + TF_DISALLOW_COPY_AND_ASSIGN(DeltaDelta); +}; // class DeltaDelta + +} // namespace delta + +#endif // DELTA_LAYERS_OPS_KERNELS_DELTA_DELTA_H_ diff --git a/athena/transform/feats/ops/kernels/delta_delta_op.cc b/athena/transform/feats/ops/kernels/delta_delta_op.cc new file mode 100644 index 00000000..c8ce3936 --- /dev/null +++ b/athena/transform/feats/ops/kernels/delta_delta_op.cc @@ -0,0 +1,84 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// See docs in ../ops/audio_ops.cc +#include "kernels/delta_delta.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/status.h" + +// https://github.com/eigenteam/eigen-git-mirror/blob/master/unsupported/Eigen/CXX11/src/Tensor/README.md + +namespace delta { + +// add first and seoncd order derivatives +class DeltaDeltaOp : public OpKernel { + public: + explicit DeltaDeltaOp(OpKernelConstruction* context) : OpKernel(context) { + OP_REQUIRES_OK(context, context->GetAttr("order", &order_)); + OP_REQUIRES_OK(context, context->GetAttr("window", &window_)); + } + + void Compute(OpKernelContext* context) override { + const Tensor& feats = context->input(0); + OP_REQUIRES(context, feats.dims() == 2, + errors::InvalidArgument("features must be 2-dimensional", + feats.shape().DebugString())); + // feats shape [time, feat dim] + const int time = feats.dim_size(0); // num frames + const int feat_dim = feats.dim_size(1); + const int output_dim = feat_dim * (order_ + 1); + + DeltaDelta delta; + OP_REQUIRES( + context, delta.Initialize(order_, window_), + errors::InvalidArgument("DeltaDelta initialization failed for order ", + order_, " and window ", window_)); + + Tensor* output_tensor = nullptr; + OP_REQUIRES_OK(context, + context->allocate_output(0, TensorShape({time, output_dim}), + &output_tensor)); + + // TType::Tensor feats_t = feats.tensor; + float* output_flat = output_tensor->flat().data(); + + for (int t = 0; t < time; t++) { + float* row = output_flat + t * output_dim; + + // add delta-delta + std::vector out; + delta.Compute(feats, t, &out); + + // fill output buffer + DCHECK_EQ(output_dim, out.size()); + for (int i = 0; i < output_dim; i++) { + row[i] = static_cast(out[i]); + } + } + } + + private: + int order_; + int window_; +}; + +REGISTER_KERNEL_BUILDER(Name("DeltaDelta").Device(DEVICE_CPU), DeltaDeltaOp); + +} // namespace delta diff --git a/athena/transform/feats/ops/kernels/delta_delta_op_test.py b/athena/transform/feats/ops/kernels/delta_delta_op_test.py new file mode 100644 index 00000000..bccfc602 --- /dev/null +++ b/athena/transform/feats/ops/kernels/delta_delta_op_test.py @@ -0,0 +1,302 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" delta-delta op unittest""" +import tempfile +import numpy as np +import tensorflow as tf +from absl import logging +from kaldiio import WriteHelper + +from delta.layers.ops import py_x_ops + + +class DeltaDeltaOpTest(tf.test.TestCase): + """ delta-delta op test""" + + def setUp(self): + """ set up""" + self.feat_dim = 80 + self.order = 2 + self.window = 2 + self.data = np.arange(self.feat_dim, dtype=np.float32) + + # dump to ark to computing delta-delta by kaldi + ark_file = tempfile.mktemp(suffix="feat.ark") + scp_file = tempfile.mktemp(suffix="feat.scp") + logging.info("ark, scp: {} {}".format(ark_file, scp_file)) + with WriteHelper("ark,scp:{},{}".format(ark_file, scp_file)) as writer: + writer(str(0), self.data[None, :]) + + # compute from kaldi `add-detlas` tools + self.output_true = np.array( + [ + 0.0000000e00, + 1.0000000e00, + 2.0000000e00, + 3.0000000e00, + 4.0000000e00, + 5.0000000e00, + 6.0000000e00, + 7.0000000e00, + 8.0000000e00, + 9.0000000e00, + 1.0000000e01, + 1.1000000e01, + 1.2000000e01, + 1.3000000e01, + 1.4000000e01, + 1.5000000e01, + 1.6000000e01, + 1.7000000e01, + 1.8000000e01, + 1.9000000e01, + 2.0000000e01, + 2.1000000e01, + 2.2000000e01, + 2.3000000e01, + 2.4000000e01, + 2.5000000e01, + 2.6000000e01, + 2.7000000e01, + 2.8000000e01, + 2.9000000e01, + 3.0000000e01, + 3.1000000e01, + 3.2000000e01, + 3.3000000e01, + 3.4000000e01, + 3.5000000e01, + 3.6000000e01, + 3.7000000e01, + 3.8000000e01, + 3.9000000e01, + 4.0000000e01, + 4.1000000e01, + 4.2000000e01, + 4.3000000e01, + 4.4000000e01, + 4.5000000e01, + 4.6000000e01, + 4.7000000e01, + 4.8000000e01, + 4.9000000e01, + 5.0000000e01, + 5.1000000e01, + 5.2000000e01, + 5.3000000e01, + 5.4000000e01, + 5.5000000e01, + 5.6000000e01, + 5.7000000e01, + 5.8000000e01, + 5.9000000e01, + 6.0000000e01, + 6.1000000e01, + 6.2000000e01, + 6.3000000e01, + 6.4000000e01, + 6.5000000e01, + 6.6000000e01, + 6.7000000e01, + 6.8000000e01, + 6.9000000e01, + 7.0000000e01, + 7.1000000e01, + 7.2000000e01, + 7.3000000e01, + 7.4000000e01, + 7.5000000e01, + 7.6000000e01, + 7.7000000e01, + 7.8000000e01, + 7.9000000e01, + 0.0000000e00, + -1.4901161e-08, + -2.9802322e-08, + 0.0000000e00, + -5.9604645e-08, + 0.0000000e00, + 0.0000000e00, + 1.1920929e-07, + -1.1920929e-07, + 1.1920929e-07, + 0.0000000e00, + -2.3841858e-07, + 0.0000000e00, + 2.3841858e-07, + 2.3841858e-07, + 0.0000000e00, + -2.3841858e-07, + -2.3841858e-07, + 2.3841858e-07, + 2.3841858e-07, + 0.0000000e00, + 4.7683716e-07, + -4.7683716e-07, + 4.7683716e-07, + 0.0000000e00, + 0.0000000e00, + 4.7683716e-07, + -4.7683716e-07, + 4.7683716e-07, + -4.7683716e-07, + 0.0000000e00, + 4.7683716e-07, + -4.7683716e-07, + 4.7683716e-07, + -4.7683716e-07, + 0.0000000e00, + 4.7683716e-07, + -4.7683716e-07, + 4.7683716e-07, + -4.7683716e-07, + 0.0000000e00, + 9.5367432e-07, + 9.5367432e-07, + 0.0000000e00, + -9.5367432e-07, + 0.0000000e00, + 9.5367432e-07, + 9.5367432e-07, + 0.0000000e00, + -9.5367432e-07, + 0.0000000e00, + 9.5367432e-07, + 9.5367432e-07, + 0.0000000e00, + -9.5367432e-07, + 0.0000000e00, + 9.5367432e-07, + 9.5367432e-07, + -9.5367432e-07, + -9.5367432e-07, + 0.0000000e00, + 9.5367432e-07, + 9.5367432e-07, + -9.5367432e-07, + -9.5367432e-07, + 0.0000000e00, + 9.5367432e-07, + 9.5367432e-07, + -9.5367432e-07, + -9.5367432e-07, + 0.0000000e00, + 9.5367432e-07, + 9.5367432e-07, + -9.5367432e-07, + -9.5367432e-07, + 0.0000000e00, + 9.5367432e-07, + 9.5367432e-07, + -9.5367432e-07, + -9.5367432e-07, + 0.0000000e00, + 0.0000000e00, + 0.0000000e00, + 0.0000000e00, + 0.0000000e00, + 5.9604645e-08, + 0.0000000e00, + 5.9604645e-08, + 0.0000000e00, + 0.0000000e00, + 1.1920929e-07, + 5.9604645e-08, + 0.0000000e00, + 0.0000000e00, + 1.1920929e-07, + 0.0000000e00, + 0.0000000e00, + 2.3841858e-07, + 0.0000000e00, + 2.3841858e-07, + 2.3841858e-07, + 0.0000000e00, + 1.1920929e-07, + 2.3841858e-07, + 0.0000000e00, + 2.3841858e-07, + 0.0000000e00, + 0.0000000e00, + 2.3841858e-07, + 0.0000000e00, + 0.0000000e00, + 0.0000000e00, + 0.0000000e00, + 0.0000000e00, + 4.7683716e-07, + 0.0000000e00, + 0.0000000e00, + 4.7683716e-07, + 4.7683716e-07, + 2.3841858e-07, + 4.7683716e-07, + 4.7683716e-07, + 0.0000000e00, + 0.0000000e00, + 2.3841858e-07, + 0.0000000e00, + 4.7683716e-07, + 2.3841858e-07, + 0.0000000e00, + 4.7683716e-07, + 4.7683716e-07, + 9.5367432e-07, + 0.0000000e00, + 4.7683716e-07, + 0.0000000e00, + 4.7683716e-07, + 4.7683716e-07, + 4.7683716e-07, + 0.0000000e00, + 4.7683716e-07, + 0.0000000e00, + 4.7683716e-07, + 0.0000000e00, + 4.7683716e-07, + 0.0000000e00, + 4.7683716e-07, + 0.0000000e00, + 4.7683716e-07, + 9.5367432e-07, + 4.7683716e-07, + 0.0000000e00, + 4.7683716e-07, + 0.0000000e00, + 4.7683716e-07, + 9.5367432e-07, + 4.7683716e-07, + 9.5367432e-07, + 0.0000000e00, + 4.7683716e-07, + 4.7683716e-07, + ], + dtype=np.float32, + ) + + def test_detla_delta(self): + """ test delta delta""" + with self.session(): + feat = tf.constant(self.data[None, :], dtype=tf.float32) + output = py_x_ops.delta_delta(feat, order=self.order, window=self.window) + self.assertEqual(tf.rank(output).eval(), tf.rank(feat).eval()) + self.assertEqual(output.shape, (1, self.feat_dim * (self.order + 1))) + self.assertAllClose(output.eval(), self.output_true[None, :]) + + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + tf.test.main() diff --git a/athena/transform/feats/ops/kernels/fbank.cc b/athena/transform/feats/ops/kernels/fbank.cc new file mode 100644 index 00000000..0427443f --- /dev/null +++ b/athena/transform/feats/ops/kernels/fbank.cc @@ -0,0 +1,70 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "kernels/fbank.h" + +#include + +#include "tensorflow/core/platform/logging.h" + +namespace delta { + +const double kDefaultUpperFrequencyLimit = 4000; +const double kDefaultLowerFrequencyLimit = 20; +const double kFilterbankFloor = 1e-12; +const int kDefaultFilterbankChannelCount = 40; + +Fbank::Fbank() + : initialized_(false), + lower_frequency_limit_(kDefaultLowerFrequencyLimit), + upper_frequency_limit_(kDefaultUpperFrequencyLimit), + filterbank_channel_count_(kDefaultFilterbankChannelCount) {} + +Fbank::~Fbank() {} + +bool Fbank::Initialize(int input_length, double input_sample_rate) { + if (input_length < 1) { + LOG(ERROR) << "Input length must be positive."; + return false; + } + input_length_ = input_length; + + bool initialized = mel_filterbank_.Initialize( + input_length, input_sample_rate, filterbank_channel_count_, + lower_frequency_limit_, upper_frequency_limit_); + initialized_ = initialized; + return initialized; +} + +void Fbank::Compute(const std::vector& spectrogram_frame, + std::vector* output) const { + if (!initialized_) { + LOG(ERROR) << "Fbank not initialized."; + return; + } + + output->resize(filterbank_channel_count_); + mel_filterbank_.Compute(spectrogram_frame, output); + for (int i = 0; i < output->size(); ++i) { + double val = (*output)[i]; + if (val < kFilterbankFloor) { + val = kFilterbankFloor; + } + (*output)[i] = log(val); + } +} + +} // namespace delta diff --git a/athena/transform/feats/ops/kernels/fbank.h b/athena/transform/feats/ops/kernels/fbank.h new file mode 100644 index 00000000..d286c1af --- /dev/null +++ b/athena/transform/feats/ops/kernels/fbank.h @@ -0,0 +1,72 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef DELTA_LAYERS_OPS_KERNELS_FBANK_H_ +#define DELTA_LAYERS_OPS_KERNELS_FBANK_H_ + +#include // NOLINT + +#include "kernels/mfcc_mel_filterbank.h" + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/platform/logging.h" + +using namespace tensorflow; // NOLINT + +namespace delta { + +// logfbank +class Fbank { + public: + Fbank(); + ~Fbank(); + bool Initialize(int input_length, double input_sample_rate); + // Input is a single squared-magnitude spectrogram frame. The input spectrum + // is converted to linear magnitude and weighted into bands using a + // triangular mel filterbank. Output is populated with the lowest + // fbank_channel_count + // of these values. + void Compute(const std::vector& spectrogram_frame, + std::vector* output) const; + + void set_upper_frequency_limit(double upper_frequency_limit) { + CHECK(!initialized_) << "Set frequency limits before calling Initialize."; + upper_frequency_limit_ = upper_frequency_limit; + } + + void set_lower_frequency_limit(double lower_frequency_limit) { + CHECK(!initialized_) << "Set frequency limits before calling Initialize."; + lower_frequency_limit_ = lower_frequency_limit; + } + + void set_filterbank_channel_count(int filterbank_channel_count) { + CHECK(!initialized_) << "Set channel count before calling Initialize."; + filterbank_channel_count_ = filterbank_channel_count; + } + + private: + MfccMelFilterbank mel_filterbank_; + int input_length_; + bool initialized_; + double lower_frequency_limit_; + double upper_frequency_limit_; + int filterbank_channel_count_; + TF_DISALLOW_COPY_AND_ASSIGN(Fbank); +}; // class Fbank + +} // namespace delta + +#endif // DELTA_LAYERS_OPS_KERNELS_FBANK_H_ diff --git a/athena/transform/feats/ops/kernels/fbank_op.cc b/athena/transform/feats/ops/kernels/fbank_op.cc new file mode 100644 index 00000000..a766e22b --- /dev/null +++ b/athena/transform/feats/ops/kernels/fbank_op.cc @@ -0,0 +1,116 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// See docs in ../ops/audio_ops.cc +#include "kernels/fbank.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/status.h" + +namespace delta { + +// Create a speech fingerpring from spectrogram data. +class FbankOp : public OpKernel { + public: + explicit FbankOp(OpKernelConstruction* context) : OpKernel(context) { + OP_REQUIRES_OK(context, context->GetAttr("upper_frequency_limit", + &upper_frequency_limit_)); + OP_REQUIRES_OK(context, context->GetAttr("lower_frequency_limit", + &lower_frequency_limit_)); + OP_REQUIRES_OK(context, context->GetAttr("filterbank_channel_count", + &filterbank_channel_count_)); + } + + void Compute(OpKernelContext* context) override { + const Tensor& spectrogram = context->input(0); + OP_REQUIRES(context, spectrogram.dims() == 3, + errors::InvalidArgument("spectrogram must be 3-dimensional", + spectrogram.shape().DebugString())); + const Tensor& sample_rate_tensor = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(sample_rate_tensor.shape()), + errors::InvalidArgument( + "Input sample_rate should be a scalar tensor, got ", + sample_rate_tensor.shape().DebugString(), " instead.")); + const int32 sample_rate = sample_rate_tensor.scalar()(); + + if (upper_frequency_limit_ <= 0) + upper_frequency_limit_ = sample_rate / 2.0 + upper_frequency_limit_; + else if (upper_frequency_limit_ > sample_rate / 2.0 || upper_frequency_limit_ <= lower_frequency_limit_) + upper_frequency_limit_ = sample_rate / 2.0; + + // shape [channels, time, bins] + const int spectrogram_channels = spectrogram.dim_size(2); + const int spectrogram_samples = spectrogram.dim_size(1); + const int audio_channels = spectrogram.dim_size(0); + + Fbank fbank; + fbank.set_upper_frequency_limit(upper_frequency_limit_); + fbank.set_lower_frequency_limit(lower_frequency_limit_); + fbank.set_filterbank_channel_count(filterbank_channel_count_); + OP_REQUIRES(context, fbank.Initialize(spectrogram_channels, sample_rate), + errors::InvalidArgument( + "Fbank initialization failed for channel count ", + spectrogram_channels, " and sample rate ", sample_rate)); + + Tensor* output_tensor = nullptr; + OP_REQUIRES_OK(context, + context->allocate_output( + 0, + TensorShape({audio_channels, spectrogram_samples, + filterbank_channel_count_}), + &output_tensor)); + + const float* spectrogram_flat = spectrogram.flat().data(); + float* output_flat = output_tensor->flat().data(); + + for (int audio_channel = 0; audio_channel < audio_channels; + ++audio_channel) { + for (int spectrogram_sample = 0; spectrogram_sample < spectrogram_samples; + ++spectrogram_sample) { + const float* sample_data = + spectrogram_flat + + (audio_channel * spectrogram_samples * spectrogram_channels) + + (spectrogram_sample * spectrogram_channels); + std::vector fbank_input(sample_data, + sample_data + spectrogram_channels); + std::vector fbank_output; + fbank.Compute(fbank_input, &fbank_output); + DCHECK_EQ(filterbank_channel_count_, fbank_output.size()); + float* output_data = + output_flat + + (audio_channel * spectrogram_samples * filterbank_channel_count_) + + (spectrogram_sample * filterbank_channel_count_); + for (int i = 0; i < filterbank_channel_count_; ++i) { + output_data[i] = fbank_output[i]; + } + std::vector().swap(fbank_input); + std::vector().swap(fbank_output); + } + } + } + + private: + float upper_frequency_limit_; + float lower_frequency_limit_; + int32 filterbank_channel_count_; +}; + +REGISTER_KERNEL_BUILDER(Name("Fbank").Device(DEVICE_CPU), FbankOp); + +} // namespace delta diff --git a/athena/transform/feats/ops/kernels/fbank_op_test.py b/athena/transform/feats/ops/kernels/fbank_op_test.py new file mode 100644 index 00000000..b30963ca --- /dev/null +++ b/athena/transform/feats/ops/kernels/fbank_op_test.py @@ -0,0 +1,72 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" fbank op unittest""" +import numpy as np +import tensorflow as tf + +from delta.layers.ops import py_x_ops + + +class FbankOpTest(tf.test.TestCase): + """ fbank op unittest""" + + def setUp(self): + """ setup """ + + def tearDown(self): + """ tear donw """ + + def test_fbank(self): + """ test fbank op""" + with self.session(): + data = np.arange(513) + spectrogram = tf.constant(data[None, None, :], dtype=tf.float32) + sample_rate = tf.constant(22050, tf.int32) + output = py_x_ops.fbank( + spectrogram, sample_rate, filterbank_channel_count=20 + ) + + output_true = np.array( + [ + 1.887894, + 2.2693727, + 2.576507, + 2.8156495, + 3.036504, + 3.2296343, + 3.4274294, + 3.5987632, + 3.771217, + 3.937401, + 4.0988584, + 4.2570987, + 4.4110703, + 4.563661, + 4.7140336, + 4.8626432, + 5.009346, + 5.1539173, + 5.2992935, + 5.442024, + ] + ) + self.assertEqual(tf.rank(output).eval(), 3) + self.assertEqual(output.shape, (1, 1, 20)) + self.assertAllClose(output.eval(), output_true[None, None, :]) + + +if __name__ == "__main__": + tf.test.main() diff --git a/athena/transform/feats/ops/kernels/framepow.cc b/athena/transform/feats/ops/kernels/framepow.cc new file mode 100644 index 00000000..4fc13f2f --- /dev/null +++ b/athena/transform/feats/ops/kernels/framepow.cc @@ -0,0 +1,115 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "kernels/framepow.h" +#include +#include +#include +#include + +namespace delta { +const float window_length_sec = 0.025; +const float frame_length_sec = 0.010; + +FramePow::FramePow() { + window_length_sec_ = window_length_sec; + frame_length_sec_ = frame_length_sec; + i_snip_edges = 1; + i_remove_dc_offset = true; + pf_FrmEng = NULL; +} + +FramePow::~FramePow() { free(pf_FrmEng); } + +void FramePow::set_window_length_sec(float window_length_sec) { + window_length_sec_ = window_length_sec; +} + +void FramePow::set_frame_length_sec(float frame_length_sec) { + frame_length_sec_ = frame_length_sec; +} + +void FramePow::set_snip_edges(int snip_edges) { i_snip_edges = snip_edges; } + +void FramePow::set_remove_dc_offset(bool remove_dc_offset) { + i_remove_dc_offset = remove_dc_offset; + } + +int FramePow::init_eng(int input_size, float sample_rate) { + f_SamRat = sample_rate; + i_WinLen = static_cast(window_length_sec_ * f_SamRat); + i_FrmLen = static_cast(frame_length_sec_ * f_SamRat); + if (i_snip_edges == 1) + i_NumFrm = (input_size - i_WinLen) / i_FrmLen + 1; + else + i_NumFrm = (input_size + i_FrmLen / 2) / i_FrmLen; + + pf_FrmEng = static_cast(malloc(sizeof(float) * i_NumFrm)); + + return 1; +} + +int FramePow::proc_eng(const float* mic_buf, int input_size) { + int i, n, k; + float* win = static_cast(malloc(sizeof(float) * i_WinLen)); + + for (n = 0; n < i_NumFrm; n++) { + pf_FrmEng[n] = 0.0; + float sum = 0.0; + float energy = 0.0; + for (k = 0; k < i_WinLen; k++) { + int index = n * i_FrmLen + k; + if (index < input_size) + win[k] = mic_buf[index]; + else + win[k] = 0.0f; + sum += win[k]; + } + + if (i_remove_dc_offset == true) { + float mean = sum / i_WinLen; + for (int l = 0; l < i_WinLen; l++) win[l] -= mean; + } + + for (i = 0; i < i_WinLen; i++) { + energy += win[i] * win[i]; + } + + pf_FrmEng[n] = log(energy); + + } + + free(win); + return 1; +} + +int FramePow::get_eng(float* output) { + memcpy(output, pf_FrmEng, sizeof(float) * i_NumFrm); + + return 1; +} + +int FramePow::write_eng() { + FILE* fp; + fp = fopen("frame_energy.txt", "w"); + int n; + for (n = 0; n < i_NumFrm; n++) { + fprintf(fp, "%4.6f\n", pf_FrmEng[n]); + } + fclose(fp); + return 1; +} +} // namespace delta diff --git a/athena/transform/feats/ops/kernels/framepow.h b/athena/transform/feats/ops/kernels/framepow.h new file mode 100644 index 00000000..c756da78 --- /dev/null +++ b/athena/transform/feats/ops/kernels/framepow.h @@ -0,0 +1,64 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef DELTA_LAYERS_OPS_KERNELS_FRAMEPOW_H_ +#define DELTA_LAYERS_OPS_KERNELS_FRAMEPOW_H_ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/platform/logging.h" + +using namespace tensorflow; // NOLINT + +namespace delta { +class FramePow { + private: + float window_length_sec_; + float frame_length_sec_; + int i_snip_edges; + bool i_remove_dc_offset; + + float f_SamRat; + int i_WinLen; + int i_FrmLen; + int i_NumFrm; + + float* pf_FrmEng; + + public: + FramePow(); + + ~FramePow(); + + void set_window_length_sec(float window_length_sec); + + void set_frame_length_sec(float frame_length_sec); + + void set_snip_edges(int snip_edges); + + void set_remove_dc_offset(bool remove_dc_offset); + + int init_eng(int input_size, float sample_rate); + + int proc_eng(const float* mic_buf, int input_size); + + int get_eng(float* output); + + int write_eng(); + + TF_DISALLOW_COPY_AND_ASSIGN(FramePow); +}; +} // namespace delta +#endif // DELTA_LAYERS_OPS_KERNELS_FRAMEPOW_H_ diff --git a/athena/transform/feats/ops/kernels/framepow_op.cc b/athena/transform/feats/ops/kernels/framepow_op.cc new file mode 100644 index 00000000..6c2869c4 --- /dev/null +++ b/athena/transform/feats/ops/kernels/framepow_op.cc @@ -0,0 +1,88 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "kernels/framepow.h" + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/status.h" + +namespace delta { +class FramePowOp : public OpKernel { + public: + explicit FramePowOp(OpKernelConstruction* context) : OpKernel(context) { + OP_REQUIRES_OK(context, context->GetAttr("window_length", &window_length_)); + OP_REQUIRES_OK(context, context->GetAttr("frame_length", &frame_length_)); + OP_REQUIRES_OK(context, context->GetAttr("snip_edges", &snip_edges_)); + OP_REQUIRES_OK(context, + context->GetAttr("remove_dc_offset", &remove_dc_offset_)); + } + + void Compute(OpKernelContext* context) override { + const Tensor& input_tensor = context->input(0); + OP_REQUIRES(context, input_tensor.dims() == 1, + errors::InvalidArgument("input signal must be 1-dimensional", + input_tensor.shape().DebugString())); + + const Tensor& sample_rate_tensor = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(sample_rate_tensor.shape()), + errors::InvalidArgument( + "Input sample rate should be a scalar tensor, got ", + sample_rate_tensor.shape().DebugString(), " instead.")); + const float sample_rate = sample_rate_tensor.scalar()(); + + // shape + const int L = input_tensor.dim_size(0); + FramePow cls_eng; + cls_eng.set_window_length_sec(window_length_); + cls_eng.set_frame_length_sec(frame_length_); + cls_eng.set_snip_edges(snip_edges_); + cls_eng.set_remove_dc_offset(remove_dc_offset_); + OP_REQUIRES(context, cls_eng.init_eng(L, sample_rate), + errors::InvalidArgument( + "framepow_class initialization failed for length ", L, + " and sample rate ", sample_rate)); + + Tensor* output_tensor = nullptr; + int i_WinLen = static_cast(window_length_ * sample_rate); + int i_FrmLen = static_cast(frame_length_ * sample_rate); + int i_NumFrm = (L - i_WinLen) / i_FrmLen + 1; + if (snip_edges_ == 2) i_NumFrm = (L + i_FrmLen / 2) / i_FrmLen; + if (i_NumFrm < 1) i_NumFrm = 1; + OP_REQUIRES_OK(context, context->allocate_output( + 0, TensorShape({1, i_NumFrm}), &output_tensor)); + + const float* input_flat = input_tensor.flat().data(); + float* output_flat = output_tensor->flat().data(); + + int ret; + ret = cls_eng.proc_eng(input_flat, L); + ret = cls_eng.get_eng(output_flat); + } + + private: + float window_length_; + float frame_length_; + int snip_edges_; + bool remove_dc_offset_; +}; + +REGISTER_KERNEL_BUILDER(Name("FramePow").Device(DEVICE_CPU), FramePowOp); + +} // namespace delta \ No newline at end of file diff --git a/athena/transform/feats/ops/kernels/mfcc_dct.cc b/athena/transform/feats/ops/kernels/mfcc_dct.cc new file mode 100644 index 00000000..43c8320a --- /dev/null +++ b/athena/transform/feats/ops/kernels/mfcc_dct.cc @@ -0,0 +1,106 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "kernels/mfcc_dct.h" + +#include + +#include "tensorflow/core/platform/logging.h" + +namespace delta { + +const float kDefaultCepstralLifter = 22; +const int kDefaultCoefficientCount = 13; + +MfccDct::MfccDct() + : initialized_(false), + coefficient_count_(kDefaultCoefficientCount), + cepstral_lifter_(kDefaultCepstralLifter) {} + +bool MfccDct::Initialize(int input_length, int coefficient_count) { + coefficient_count_ = coefficient_count; + input_length_ = input_length; + + if (coefficient_count_ < 1) { + LOG(ERROR) << "Coefficient count must be positive."; + return false; + } + + if (input_length < 1) { + LOG(ERROR) << "Input length must be positive."; + return false; + } + + if (coefficient_count_ > input_length_) { + LOG(ERROR) << "Coefficient count must be less than or equal to " + << "input length."; + return false; + } + + cosines_.resize(coefficient_count_); + double fnorm = sqrt(2.0 / input_length_); + // Some platforms don't have M_PI, so define a local constant here. + const double pi = std::atan(1) * 4; + double arg = pi / input_length_; + for (int i = 0; i < coefficient_count_; ++i) { + cosines_[i].resize(input_length_); + for (int j = 0; j < input_length_; ++j) { + cosines_[i][j] = fnorm * cos(i * arg * (j + 0.5)); + } + } + + lifter_coeffs_.resize(coefficient_count_); + for (int j = 0; j < coefficient_count_; ++j) + lifter_coeffs_[j] = + 1.0 + 0.5 * cepstral_lifter_ * sin(PI * j / cepstral_lifter_); + + initialized_ = true; + return true; +} + +void MfccDct::set_coefficient_count(int coefficient_count) { + coefficient_count_ = coefficient_count; +} + +void MfccDct::set_cepstral_lifter(float cepstral_lifter) { + cepstral_lifter_ = cepstral_lifter; +} + +void MfccDct::Compute(const std::vector &input, + std::vector *output) const { + if (!initialized_) { + LOG(ERROR) << "DCT not initialized."; + return; + } + + output->resize(coefficient_count_); + int length = input.size(); + if (length > input_length_) { + length = input_length_; + } + + double res; + for (int i = 0; i < coefficient_count_; ++i) { + double sum = 0.0; + for (int j = 0; j < length; ++j) { + sum += cosines_[i][j] * input[j]; + } + res = sum; + if (cepstral_lifter_ != 0) res *= lifter_coeffs_[i]; + (*output)[i] = res; + } +} + +} // namespace delta diff --git a/athena/transform/feats/ops/kernels/mfcc_dct.h b/athena/transform/feats/ops/kernels/mfcc_dct.h new file mode 100644 index 00000000..16e76e38 --- /dev/null +++ b/athena/transform/feats/ops/kernels/mfcc_dct.h @@ -0,0 +1,53 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Basic minimal DCT class for MFCC speech processing. + +#ifndef TENSORFLOW_CORE_KERNELS_MFCC_DCT_H_ // NOLINT +#define TENSORFLOW_CORE_KERNELS_MFCC_DCT_H_ // NOLINT + +#include + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/platform/logging.h" +#include "kernels/support_functions.h" + +using namespace tensorflow; // NOLINT +#define PI (3.141592653589793) + +namespace delta { + +class MfccDct { + public: + MfccDct(); + bool Initialize(int input_length, int coefficient_count); + void Compute(const std::vector& input, + std::vector* output) const; + void set_coefficient_count(int coefficient_count); + void set_cepstral_lifter(float cepstral_lifter); + + private: + bool initialized_; + int coefficient_count_; + float cepstral_lifter_; + int input_length_; + std::vector > cosines_; + std::vector lifter_coeffs_; + TF_DISALLOW_COPY_AND_ASSIGN(MfccDct); +}; + +} // namespace delta + +#endif // DELTA_LAYERS_OPS_KERNELS_MFCC_DCT_H_ diff --git a/athena/transform/feats/ops/kernels/mfcc_dct_op.cc b/athena/transform/feats/ops/kernels/mfcc_dct_op.cc new file mode 100644 index 00000000..8d94b0db --- /dev/null +++ b/athena/transform/feats/ops/kernels/mfcc_dct_op.cc @@ -0,0 +1,119 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// See docs in ../ops/audio_ops.cc +#include "kernels/mfcc_dct.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/status.h" + +namespace delta { + +class MfccDctOp : public OpKernel { + public: + explicit MfccDctOp(OpKernelConstruction* context) : OpKernel(context) { + OP_REQUIRES_OK(context, + context->GetAttr("coefficient_count", &coefficient_count_)); + OP_REQUIRES_OK(context, + context->GetAttr("cepstral_lifter", &cepstral_lifter_)); + OP_REQUIRES_OK(context, + context->GetAttr("use_energy", &use_energy_)); + } + + void Compute(OpKernelContext* context) override { + const Tensor& fbank = context->input(0); + OP_REQUIRES(context, fbank.dims() == 3, + errors::InvalidArgument("Fbank must be 3-dimensional", + fbank.shape().DebugString())); + const Tensor& framepow = context->input(1); + OP_REQUIRES(context, framepow.dims() == 1, + errors::InvalidArgument("Framepow must be 1-dimensional", + framepow.shape().DebugString())); + const Tensor& sample_rate_tensor = context->input(2); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(sample_rate_tensor.shape()), + errors::InvalidArgument( + "Input sample_rate should be a scalar tensor, got ", + sample_rate_tensor.shape().DebugString(), " instead.")); + const int32 sample_rate = sample_rate_tensor.scalar()(); + + // shape [channels, time, bins] + const int fbank_channels = fbank.dim_size(2); + const int fbank_samples = fbank.dim_size(1); + const int audio_channels = fbank.dim_size(0); + + MfccDct mfcc; + mfcc.set_coefficient_count(coefficient_count_); + mfcc.set_cepstral_lifter(cepstral_lifter_); + + OP_REQUIRES( + context, mfcc.Initialize(fbank_channels, coefficient_count_), + errors::InvalidArgument("MFCC initialization failed for fbank channel ", + fbank_channels, " and coefficient count", + coefficient_count_)); + + Tensor* output_tensor = nullptr; + OP_REQUIRES_OK( + context, + context->allocate_output( + 0, TensorShape({audio_channels, fbank_samples, coefficient_count_}), + &output_tensor)); + + const float* fbank_flat = fbank.flat().data(); + const float* framepow_flat = framepow.flat().data(); + float* output_flat = output_tensor->flat().data(); + + for (int audio_channel = 0; audio_channel < audio_channels; + ++audio_channel) { + for (int fbank_sample = 0; fbank_sample < fbank_samples; ++fbank_sample) { + const float* sample_data = + fbank_flat + (audio_channel * fbank_samples * fbank_channels) + + (fbank_sample * fbank_channels); + const float* framepow_data = framepow_flat + fbank_sample; + std::vector mfcc_input(sample_data, + sample_data + fbank_channels); + std::vector framepow_input(framepow_data, framepow_data + 1); + std::vector mfcc_output; + mfcc.Compute(mfcc_input, &mfcc_output); + DCHECK_EQ(coefficient_count_, mfcc_output.size()); + float* output_data = + output_flat + (audio_channel * fbank_samples * coefficient_count_) + + (fbank_sample * coefficient_count_); + for (int i = 0; i < coefficient_count_; ++i) { + output_data[i] = mfcc_output[i]; + } + if (use_energy_) + output_data[0] = framepow_input[0]; + + std::vector().swap(mfcc_input); + std::vector().swap(framepow_input); + std::vector().swap(mfcc_output); + } + } + + } + + private: + float cepstral_lifter_; + int coefficient_count_; + bool use_energy_; +}; + +REGISTER_KERNEL_BUILDER(Name("MfccDct").Device(DEVICE_CPU), MfccDctOp); + +} // namespace delta diff --git a/athena/transform/feats/ops/kernels/mfcc_mel_filterbank.cc b/athena/transform/feats/ops/kernels/mfcc_mel_filterbank.cc new file mode 100644 index 00000000..46c0c193 --- /dev/null +++ b/athena/transform/feats/ops/kernels/mfcc_mel_filterbank.cc @@ -0,0 +1,211 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This code resamples the FFT bins, and smooths then with triangle-shaped +// weights to create a mel-frequency filter bank. For filter i centered at f_i, +// there is a triangular weighting of the FFT bins that extends from +// filter f_i-1 (with a value of zero at the left edge of the triangle) to f_i +// (where the filter value is 1) to f_i+1 (where the filter values returns to +// zero). + +// Note: this code fails if you ask for too many channels. The algorithm used +// here assumes that each FFT bin contributes to at most two channels: the +// right side of a triangle for channel i, and the left side of the triangle +// for channel i+1. If you ask for so many channels that some of the +// resulting mel triangle filters are smaller than a single FFT bin, these +// channels may end up with no contributing FFT bins. The resulting mel +// spectrum output will have some channels that are always zero. + +#include "kernels/mfcc_mel_filterbank.h" + +#include + +#include "tensorflow/core/platform/logging.h" + +namespace tensorflow { + +MfccMelFilterbank::MfccMelFilterbank() : initialized_(false) {} + +MfccMelFilterbank::~MfccMelFilterbank() { + std::vector().swap(center_frequencies_); + std::vector().swap(weights_); + std::vector().swap(band_mapper_); +} + +bool MfccMelFilterbank::Initialize(int input_length, double input_sample_rate, + int output_channel_count, + double lower_frequency_limit, + double upper_frequency_limit) { + num_channels_ = output_channel_count; + sample_rate_ = input_sample_rate; + input_length_ = input_length; + + if (num_channels_ < 1) { + LOG(ERROR) << "Number of filterbank channels must be positive."; + return false; + } + + if (sample_rate_ <= 0) { + LOG(ERROR) << "Sample rate must be positive."; + return false; + } + + if (input_length < 2) { + LOG(ERROR) << "Input length must greater than 1."; + return false; + } + + if (lower_frequency_limit < 0) { + LOG(ERROR) << "Lower frequency limit must be nonnegative."; + return false; + } + + if (upper_frequency_limit <= lower_frequency_limit) { + LOG(ERROR) << "Upper frequency limit must be greater than " + << "lower frequency limit."; + return false; + } + + // An extra center frequency is computed at the top to get the upper + // limit on the high side of the final triangular filter. + center_frequencies_.resize(num_channels_ + 1); + const double mel_low = FreqToMel(lower_frequency_limit); + const double mel_hi = FreqToMel(upper_frequency_limit); + const double mel_span = mel_hi - mel_low; + const double mel_spacing = mel_span / static_cast(num_channels_ + 1); + for (int i = 0; i < num_channels_ + 1; ++i) { + center_frequencies_[i] = mel_low + (mel_spacing * (i + 1)); + } + + // Always exclude DC; emulate HTK. + const double hz_per_sbin = + 0.5 * sample_rate_ / static_cast(input_length_ - 1); + start_index_ = static_cast(1 + (lower_frequency_limit / hz_per_sbin)); + end_index_ = static_cast(upper_frequency_limit / hz_per_sbin); + + // Maps the input spectrum bin indices to filter bank channels/indices. For + // each FFT bin, band_mapper tells us which channel this bin contributes to + // on the right side of the triangle. Thus this bin also contributes to the + // left side of the next channel's triangle response. + band_mapper_.resize(input_length_); + int channel = 0; + for (int i = 0; i < input_length_; ++i) { + double melf = FreqToMel(i * hz_per_sbin); + if ((i < start_index_) || (i > end_index_)) { + band_mapper_[i] = -2; // Indicate an unused Fourier coefficient. + } else { + while ((center_frequencies_[channel] < melf) && + (channel < num_channels_)) { + ++channel; + } + band_mapper_[i] = channel - 1; // Can be == -1 + } + } + + // Create the weighting functions to taper the band edges. The contribution + // of any one FFT bin is based on its distance along the continuum between two + // mel-channel center frequencies. This bin contributes weights_[i] to the + // current channel and 1-weights_[i] to the next channel. + weights_.resize(input_length_); + for (int i = 0; i < input_length_; ++i) { + channel = band_mapper_[i]; + if ((i < start_index_) || (i > end_index_)) { + weights_[i] = 0.0; + } else { + if (channel >= 0) { + weights_[i] = + (center_frequencies_[channel + 1] - FreqToMel(i * hz_per_sbin)) / + (center_frequencies_[channel + 1] - center_frequencies_[channel]); + } else { + weights_[i] = (center_frequencies_[0] - FreqToMel(i * hz_per_sbin)) / + (center_frequencies_[0] - mel_low); + } + } + } + // Check the sum of FFT bin weights for every mel band to identify + // situations where the mel bands are so narrow that they don't get + // significant weight on enough (or any) FFT bins -- i.e., too many + // mel bands have been requested for the given FFT size. + std::vector bad_channels; + for (int c = 0; c < num_channels_; ++c) { + float band_weights_sum = 0.0; + for (int i = 0; i < input_length_; ++i) { + if (band_mapper_[i] == c - 1) { + band_weights_sum += (1.0 - weights_[i]); + } else if (band_mapper_[i] == c) { + band_weights_sum += weights_[i]; + } + } + // The lowest mel channels have the fewest FFT bins and the lowest + // weights sum. But given that the target gain at the center frequency + // is 1.0, if the total sum of weights is 0.5, we're in bad shape. + if (band_weights_sum < 0.5) { + bad_channels.push_back(c); + } + } + /* + if (!bad_channels.empty()) { + LOG(ERROR) << "Missing " << bad_channels.size() << " bands " + << " starting at " << bad_channels[0] + << " in mel-frequency design. " + << "Perhaps too many channels or " + << "not enough frequency resolution in spectrum. (" + << "input_length: " << input_length + << " input_sample_rate: " << input_sample_rate + << " output_channel_count: " << output_channel_count + << " lower_frequency_limit: " << lower_frequency_limit + << " upper_frequency_limit: " << upper_frequency_limit; + } + */ + initialized_ = true; + return true; +} + +// Compute the mel spectrum from the squared-magnitude FFT input by taking the +// square root, then summing FFT magnitudes under triangular integration windows +// whose widths increase with frequency. +void MfccMelFilterbank::Compute(const std::vector &input, + std::vector *output) const { + if (!initialized_) { + LOG(ERROR) << "Mel Filterbank not initialized."; + return; + } + + if (input.size() <= end_index_) { + LOG(ERROR) << "Input too short to compute filterbank"; + return; + } + + // Ensure output is right length and reset all values. + output->assign(num_channels_, 0.0); + + for (int i = start_index_; i <= end_index_; i++) { // For each FFT bin + double spec_val = input[i]; + double weighted = spec_val * weights_[i]; + int channel = band_mapper_[i]; + if (channel >= 0) + (*output)[channel] += weighted; // Right side of triangle, downward slope + channel++; + if (channel < num_channels_) + (*output)[channel] += spec_val - weighted; // Left side of triangle + } + +} + +double MfccMelFilterbank::FreqToMel(double freq) const { + return 1127.0 * log(1.0 + (freq / 700.0)); +} + +} // namespace tensorflow diff --git a/athena/transform/feats/ops/kernels/mfcc_mel_filterbank.h b/athena/transform/feats/ops/kernels/mfcc_mel_filterbank.h new file mode 100644 index 00000000..2a745b2d --- /dev/null +++ b/athena/transform/feats/ops/kernels/mfcc_mel_filterbank.h @@ -0,0 +1,65 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Basic class for applying a mel-scale mapping to a power spectrum. + +#ifndef DELTA_LAYERS_OPS_KERNELS_MFCC_MEL_FILTERBANK_H_ +#define DELTA_LAYERS_OPS_KERNELS_MFCC_MEL_FILTERBANK_H_ + +#include + +#include "tensorflow/core/framework/op_kernel.h" + +namespace tensorflow { + +class MfccMelFilterbank { + public: + MfccMelFilterbank(); + ~MfccMelFilterbank(); + bool Initialize(int input_length, // Number of unique FFT bins fftsize/2+1. + double input_sample_rate, int output_channel_count, + double lower_frequency_limit, double upper_frequency_limit); + + // Takes a squared-magnitude spectrogram slice as input, computes a + // triangular-mel-weighted linear-magnitude filterbank, and places the result + // in output. + void Compute(const std::vector& input, + std::vector* output) const; + + private: + double FreqToMel(double freq) const; + bool initialized_; + int num_channels_; + double sample_rate_; + int input_length_; + std::vector center_frequencies_; // In mel, for each mel channel. + + // Each FFT bin b contributes to two triangular mel channels, with + // proportion weights_[b] going into mel channel band_mapper_[b], and + // proportion (1 - weights_[b]) going into channel band_mapper_[b] + 1. + // Thus, weights_ contains the weighting applied to each FFT bin for the + // upper-half of the triangular band. + std::vector weights_; // Right-side weight for this fft bin. + + // FFT bin i contributes to the upper side of mel channel band_mapper_[i] + std::vector band_mapper_; + int start_index_; // Lowest FFT bin used to calculate mel spectrum. + int end_index_; // Highest FFT bin used to calculate mel spectrum. + + TF_DISALLOW_COPY_AND_ASSIGN(MfccMelFilterbank); +}; + +} // namespace tensorflow +#endif // DELTA_LAYERS_OPS_KERNELS_MFCC_MEL_FILTERBANK_H_ diff --git a/athena/transform/feats/ops/kernels/pitch.cc b/athena/transform/feats/ops/kernels/pitch.cc new file mode 100644 index 00000000..e71c5454 --- /dev/null +++ b/athena/transform/feats/ops/kernels/pitch.cc @@ -0,0 +1,1068 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include "kernels/pitch.h" + +namespace delta { + +BaseFloat vector_Min(const std::vector &input) { + BaseFloat min_res = std::numeric_limits::infinity(); + int length = input.size(); + int i; + for (i = 0; i + 4 <= length; i += 4){ + BaseFloat a1 = input[i], a2 = input[i+1], a3 = input[i+2], a4 = input[i+3]; + if (a1 < min_res || a2 < min_res || a3 < min_res || a4 < min_res) { + BaseFloat b1 = (a1 < a2 ? a1 : a2), b2 = (a3 < a4 ? a3 : a4); + if (b1 < min_res) min_res = b1; + if (b2 < min_res) min_res = b2; + } + } + for(; i < length; i++){ + if (min_res > input[i]){ + min_res = input[i]; + } + } + return min_res; +} + +BaseFloat vector_sum(const std::vector &input){ + BaseFloat res = 0; + for (int i = 0; i < input.size(); i++) + res += input[i]; + return res; +} + +BaseFloat NccfToPovFeature(BaseFloat n) { + if (n > 1.0) { + n = 1.0; + } else if (n < -1.0) { + n = -1.0; + } + BaseFloat f = pow((1.0001 - n), 0.15) - 1.0; + assert(f - f == 0); // check for NaN,inf. + return f; +} + +BaseFloat NccfToPov(BaseFloat n) { + BaseFloat ndash = abs(n); + if (ndash > 1.0) ndash = 1.0; // just in case it was slightly outside [-1, 1] + + BaseFloat r = -5.2 + 5.4 * exp(7.5 * (ndash - 1.0)) + 4.8 * ndash - + 2.0 * exp(-10.0 * ndash) + 4.2 * exp(20.0 * (ndash - 1.0)); + // r is the approximate log-prob-ratio of voicing, log(p/(1-p)). + BaseFloat p = 1.0 / (1 + exp(-1.0 * r)); + assert(p - p == 0); // Check for NaN/inf + return p; +} + +void ComputeCorrelation(const vector &wave, + int first_lag, int last_lag, + int nccf_window_size, + vector *inner_prod, + vector *norm_prod) { + vector zero_mean_wave(wave); + // TODO: possibly fix this, the mean normalization is done in a strange way. + vector wave_part = sub_vector(wave, 0, nccf_window_size); + // subtract mean-frame from wave + BaseFloat wave_sum = 0.0; + for (int i = 0; i < wave_part.size(); i++) + wave_sum += wave_part[i]; + BaseFloat wave_mean = wave_sum / nccf_window_size; + for (int j = 0; j < zero_mean_wave.size(); j++){ + zero_mean_wave[j] -= wave_mean; + } + BaseFloat e1, e2, sum; + vector sub_vec1 = sub_vector(zero_mean_wave, 0, nccf_window_size); + e1 = VecVec(sub_vec1, sub_vec1); + for (int lag = first_lag; lag <= last_lag; lag++) { + vector sub_vec2 = sub_vector(zero_mean_wave, lag, nccf_window_size); + e2 = VecVec(sub_vec2, sub_vec2); + sum = VecVec(sub_vec1, sub_vec2); + (*inner_prod)[lag - first_lag] = sum; + (*norm_prod)[lag - first_lag] = e1 * e2; + } +} + +void ComputeNccf(const vector &inner_prod, + const vector &norm_prod, + BaseFloat nccf_ballast, + vector *nccf_vec) { + assert(inner_prod.size() == norm_prod.size() && + inner_prod.size() == nccf_vec->size()); + for (int lag = 0; lag < inner_prod.size(); lag++) { + BaseFloat numerator = inner_prod[lag], + denominator = pow(norm_prod[lag] + nccf_ballast, 0.5), + nccf; + if (denominator != 0.0) { + nccf = numerator / denominator; + } else { + assert(numerator == 0.0); + nccf = 0.0; + } + assert(nccf < 1.01 && nccf > -1.01); + (*nccf_vec)[lag] = nccf; + } +} + +void SelectLags(const PitchExtractionOptions &opts, + vector *lags) { + // choose lags relative to acceptable pitch tolerance + BaseFloat min_lag = 1.0 / opts.max_f0, max_lag = 1.0 / opts.min_f0; + + std::vector tmp_lags; + for (BaseFloat lag = min_lag; lag <= max_lag; lag *= 1.0 + opts.delta_pitch) + tmp_lags.push_back(lag); + lags->resize(tmp_lags.size()); + for (int i = 0; i < tmp_lags.size(); i++) + (*lags)[i] = tmp_lags[i]; +} + + +void ComputeLocalCost(const vector &nccf_pitch, + const vector &lags, + const PitchExtractionOptions &opts, + vector *local_cost) { + // from the paper, eq. 5, local_cost = 1 - Phi(t,i)(1 - soft_min_f0 L_i) + // nccf is the nccf on this frame measured at the lags in "lags". + assert(nccf_pitch.size() == local_cost->size() && + nccf_pitch.size() == lags.size()); + // for (int i = 0; i < local_cost->size(); i++) + // (*local_cost)[i] = 1.0; + // for (int j = 0; j < local_cost->size(); j++) + // (*local_cost)[j] -= nccf_pitch[j]; + for (int k = 0; k < local_cost->size(); k++){ + (*local_cost)[k] = 1 - (1 - opts.soft_min_f0 * lags[k]) * nccf_pitch[k]; + } +} + + +class PitchFrameInfo { + public: + void Cleanup(PitchFrameInfo *prev_frame); + + void SetBestState(int best_state, + std::vector > &lag_nccf); + + int ComputeLatency(int max_latency); + + /// This function updates + bool UpdatePreviousBestState(PitchFrameInfo *prev_frame); + + /// This constructor is used for frame -1; it sets the costs to be all zeros + /// the pov_nccf's to zero and the backpointers to -1. + explicit PitchFrameInfo(int num_states); + + /// This constructor is used for subsequent frames (not -1). + PitchFrameInfo(PitchFrameInfo *prev); + + /// Record the nccf_pov value. + /// @param nccf_pov The nccf as computed for the POV computation (without ballast). + void SetNccfPov(const vector &nccf_pov); + + void ComputeBacktraces(const PitchExtractionOptions &opts, + const vector &nccf_pitch, + const vector &lags, + const vector &prev_forward_cost, + std::vector > *index_info, + vector *this_forward_cost); + private: + struct StateInfo { + /// The state index on the previous frame that is the best preceding state + /// for this state. + int backpointer; + /// the version of the NCCF we keep for the POV computation (without the + /// ballast term). + BaseFloat pov_nccf; + StateInfo(): backpointer(0), pov_nccf(0.0) { } + }; + std::vector state_info_; + /// the state index of the first entry in "state_info"; this will initially be + /// zero, but after cleanup might be nonzero. + int state_offset_; + + /// The current best state in the backtrace from the end. + int cur_best_state_; + + /// The structure for the previous frame. + PitchFrameInfo *prev_info_; +}; + + +// This constructor is used for frame -1; it sets the costs to be all zeros +// the pov_nccf's to zero and the backpointers to -1. +PitchFrameInfo::PitchFrameInfo(int num_states) + :state_info_(num_states), state_offset_(0), + cur_best_state_(-1), prev_info_(NULL) { } + + +bool pitch_use_naive_search = false; // This is used in unit-tests. + + +PitchFrameInfo::PitchFrameInfo(PitchFrameInfo *prev_info): + state_info_(prev_info->state_info_.size()), state_offset_(0), + cur_best_state_(-1), prev_info_(prev_info) { } + +void PitchFrameInfo::SetNccfPov(const vector &nccf_pov) { + int num_states = nccf_pov.size(); + assert(num_states == state_info_.size()); + for (int i = 0; i < num_states; i++){ + state_info_[i].pov_nccf = nccf_pov[i]; + } +} + +void PitchFrameInfo::ComputeBacktraces( + const PitchExtractionOptions &opts, + const vector &nccf_pitch, + const vector &lags, + const vector &prev_forward_cost_vec, + std::vector > *index_info, + vector *this_forward_cost_vec) { + int num_states = nccf_pitch.size(); + + vector local_cost(num_states, 0); + ComputeLocalCost(nccf_pitch, lags, opts, &local_cost); + + const BaseFloat delta_pitch_sq = pow(Log(1.0 + opts.delta_pitch), 2.0), + inter_frame_factor = delta_pitch_sq * opts.penalty_factor; + + const vector prev_forward_cost = prev_forward_cost_vec; + // vector this_forward_cost = *this_forward_cost_vec; + if (index_info->empty()) + index_info->resize(num_states); + + // make it a reference for more concise indexing. + std::vector > &bounds = *index_info; + + if (pitch_use_naive_search) { + // This branch is only taken in unit-testing code. + for (int i = 0; i < num_states; i++) { + BaseFloat best_cost = std::numeric_limits::infinity(); + int best_j = -1; + for (int j = 0; j < num_states; j++) { + BaseFloat this_cost = (j - i) * (j - i) * inter_frame_factor + + prev_forward_cost[j]; + if (this_cost < best_cost) { + best_cost = this_cost; + best_j = j; + } + } + (*this_forward_cost_vec)[i] = best_cost; + state_info_[i].backpointer = best_j; + } + } else { + int last_backpointer = 0; + for (int i = 0; i < num_states; i++) { + int start_j = last_backpointer; + BaseFloat best_cost = (start_j - i) * (start_j - i) * inter_frame_factor + + prev_forward_cost[start_j]; + int best_j = start_j; + + for (int j = start_j + 1; j < num_states; j++) { + BaseFloat this_cost = (j - i) * (j - i) * inter_frame_factor + + prev_forward_cost[j]; + if (this_cost < best_cost) { + best_cost = this_cost; + best_j = j; + } else { // as soon as the costs stop improving, we stop searching. + break; // this is a loose lower bound we're getting. + } + } + state_info_[i].backpointer = best_j; + (*this_forward_cost_vec)[i] = best_cost; + bounds[i].first = best_j; // this is now a lower bound on the + // backpointer. + bounds[i].second = num_states - 1; // we have no meaningful upper bound + // yet. + last_backpointer = best_j; + } + + for (int iter = 0; iter < num_states; iter++) { + bool changed = false; + if (iter % 2 == 0) { // go backwards through the states + last_backpointer = num_states - 1; + for (int i = num_states - 1; i >= 0; i--) { + int lower_bound = bounds[i].first, + upper_bound = std::min(last_backpointer, bounds[i].second); + if (upper_bound == lower_bound) { + last_backpointer = lower_bound; + continue; + } + BaseFloat best_cost = (*this_forward_cost_vec)[i]; + int best_j = state_info_[i].backpointer, initial_best_j = best_j; + + if (best_j == upper_bound) { + last_backpointer = best_j; + continue; + } + for (int j = upper_bound; j > lower_bound + 1; j--) { + BaseFloat this_cost = (j - i) * (j - i) * inter_frame_factor + + prev_forward_cost[j]; + if (this_cost < best_cost) { + best_cost = this_cost; + best_j = j; + } else { + if (best_j > j) + break; // this is a loose lower bound we're getting. + } + } + bounds[i].second = best_j; + if (best_j != initial_best_j) { + (*this_forward_cost_vec)[i] = best_cost; + state_info_[i].backpointer = best_j; + changed = true; + } + last_backpointer = best_j; + } + } else { // go forwards through the states. + last_backpointer = 0; + for (int i = 0; i < num_states; i++) { + int lower_bound = std::max(last_backpointer, bounds[i].first), + upper_bound = bounds[i].second; + if (upper_bound == lower_bound) { + last_backpointer = lower_bound; + continue; + } + BaseFloat best_cost = (*this_forward_cost_vec)[i]; + int best_j = state_info_[i].backpointer, initial_best_j = best_j; + + if (best_j == lower_bound) { + last_backpointer = best_j; + continue; + } + // Below, we have j < upper_bound because we know we've already + // evaluated that point. + for (int j = lower_bound; j < upper_bound - 1; j++) { + BaseFloat this_cost = (j - i) * (j - i) * inter_frame_factor + + prev_forward_cost[j]; + if (this_cost < best_cost) { + best_cost = this_cost; + best_j = j; + } else { + if (best_j < j) + break; // this is a loose lower bound we're getting. + } + } + // our "best_j" is now a lower bound on the backpointer. + bounds[i].first = best_j; + if (best_j != initial_best_j) { + (*this_forward_cost_vec)[i] = best_cost; + state_info_[i].backpointer = best_j; + changed = true; + } + last_backpointer = best_j; + } + } + if (!changed) + break; + } + } + // The next statement is needed due to RecomputeBacktraces: we have to + // invalidate the previously computed best-state info. + cur_best_state_ = -1; + for (int i = 0; i < this_forward_cost_vec->size(); i++) + (*this_forward_cost_vec)[i] += local_cost[i]; +} + +void PitchFrameInfo::SetBestState( + int best_state, + std::vector > &lag_nccf) { + + std::vector >::reverse_iterator iter = lag_nccf.rbegin(); + + PitchFrameInfo *this_info = this; // it will change in the loop. + while (this_info != NULL) { + PitchFrameInfo *prev_info = this_info->prev_info_; + if (best_state == this_info->cur_best_state_) + return; // no change + if (prev_info != NULL) // don't write anything for frame -1. + iter->first = best_state; + size_t state_info_index = best_state - this_info->state_offset_; + assert(state_info_index < this_info->state_info_.size()); + this_info->cur_best_state_ = best_state; + best_state = this_info->state_info_[state_info_index].backpointer; + if (prev_info != NULL) // don't write anything for frame -1. + iter->second = this_info->state_info_[state_info_index].pov_nccf; + this_info = prev_info; + if (this_info != NULL) ++iter; + } +} + +int PitchFrameInfo::ComputeLatency(int max_latency) { + if (max_latency <= 0) return 0; + + int latency = 0; + int num_states = state_info_.size(); + int min_living_state = 0, max_living_state = num_states - 1; + PitchFrameInfo *this_info = this; // it will change in the loop. + + + for (; this_info != NULL && latency < max_latency;) { + int offset = this_info->state_offset_; + assert(min_living_state >= offset && + max_living_state - offset < this_info->state_info_.size()); + min_living_state = + this_info->state_info_[min_living_state - offset].backpointer; + max_living_state = + this_info->state_info_[max_living_state - offset].backpointer; + if (min_living_state == max_living_state) { + return latency; + } + this_info = this_info->prev_info_; + if (this_info != NULL) // avoid incrementing latency for frame -1, + latency++; // as it's not a real frame. + } + return latency; +} + +void PitchFrameInfo::Cleanup(PitchFrameInfo *prev_frame) { + std::cerr << "Cleanup not implemented."; +} + +struct NccfInfo { + + vector nccf_pitch_resampled; // resampled nccf_pitch + BaseFloat avg_norm_prod; // average value of e1 * e2. + BaseFloat mean_square_energy; // mean_square energy we used when computing the + // original ballast term for + // "nccf_pitch_resampled". + + NccfInfo(BaseFloat avg_norm_prod, + BaseFloat mean_square_energy): + avg_norm_prod(avg_norm_prod), + mean_square_energy(mean_square_energy) { } +}; + +class OnlinePitchFeatureImpl { + public: + explicit OnlinePitchFeatureImpl(const PitchExtractionOptions &opts); + + int Dim() const { return 2; } + + BaseFloat FrameShiftInSeconds() const; + + int NumFramesReady() const; + + bool IsLastFrame(int frame) const; + + void GetFrame(int frame, vector *feat); + + void AcceptWaveform(BaseFloat sampling_rate, + const vector &waveform); + + void InputFinished(); + + ~OnlinePitchFeatureImpl(); + + OnlinePitchFeatureImpl(const OnlinePitchFeatureImpl &other); + + private: + + int NumFramesAvailable(int num_downsampled_samples, bool snip_edges) const; + void ExtractFrame(const vector &downsampled_wave_part, + int frame_index, + vector *window); + + void RecomputeBacktraces(); + + void UpdateRemainder(const vector &downsampled_wave_part); + PitchExtractionOptions opts_; + + // the first lag of the downsampled signal at which we measure NCCF + int nccf_first_lag_; + // the last lag of the downsampled signal at which we measure NCCF + int nccf_last_lag_; + + // The log-spaced lags at which we will resample the NCCF + vector lags_; + ArbitraryResample *nccf_resampler_; + LinearResample *signal_resampler_; + + std::vector frame_info_; + std::vector nccf_info_; + + int frames_latency_; + vector forward_cost_; + double forward_cost_remainder_; + + std::vector > lag_nccf_; + + bool input_finished_; + double signal_sumsq_; + + double signal_sum_; + + int downsampled_samples_processed_; + vector downsampled_signal_remainder_; +}; + + +OnlinePitchFeatureImpl::OnlinePitchFeatureImpl( + const PitchExtractionOptions &opts): + opts_(opts), forward_cost_remainder_(0.0), input_finished_(false), + signal_sumsq_(0.0), signal_sum_(0.0), downsampled_samples_processed_(0) { + signal_resampler_ = new LinearResample(opts.samp_freq, opts.resample_freq, + opts.lowpass_cutoff, + opts.lowpass_filter_width); + + double outer_min_lag = 1.0 / opts.max_f0 - + (opts.upsample_filter_width/(2.0 * opts.resample_freq)); + double outer_max_lag = 1.0 / opts.min_f0 + + (opts.upsample_filter_width/(2.0 * opts.resample_freq)); + nccf_first_lag_ = ceil(opts.resample_freq * outer_min_lag); + nccf_last_lag_ = floor(opts.resample_freq * outer_max_lag); + + frames_latency_ = 0; // will be set in AcceptWaveform() + + // Choose the lags at which we resample the NCCF. + SelectLags(opts, &lags_); + BaseFloat upsample_cutoff = opts.resample_freq * 0.5; + + + vector lags_offset(lags_); + for (int i = 0; i < lags_offset.size(); i++) + lags_offset[i] += (-nccf_first_lag_ / opts.resample_freq); + + int num_measured_lags = nccf_last_lag_ + 1 - nccf_first_lag_; + + nccf_resampler_ = new ArbitraryResample(num_measured_lags, opts.resample_freq, + upsample_cutoff, lags_offset, + opts.upsample_filter_width); + + // add a PitchInfo object for frame -1 (not a real frame). + frame_info_.push_back(new PitchFrameInfo(lags_.size())); + // zeroes forward_cost_; this is what we want for the fake frame -1. + forward_cost_.resize(lags_.size()); +} + + +int OnlinePitchFeatureImpl::NumFramesAvailable( + int num_downsampled_samples, bool snip_edges) const { + int frame_shift = opts_.NccfWindowShift(), + frame_length = opts_.NccfWindowSize(); + // Use the "full frame length" to compute the number + // of frames only if the input is not finished. + if (!input_finished_) + frame_length += nccf_last_lag_; + if (num_downsampled_samples < frame_length) { + return 0; + } else { + if (!snip_edges) { + if (input_finished_) { + return static_cast(num_downsampled_samples * 1.0f / + frame_shift + 0.5f); + } else { + return static_cast((num_downsampled_samples - frame_length / 2) * + 1.0f / frame_shift + 0.5f); + } + } else { + return static_cast((num_downsampled_samples - frame_length) / + frame_shift + 1); + } + } +} + +void OnlinePitchFeatureImpl::UpdateRemainder( + const vector &downsampled_wave_part) { + int num_frames = static_cast(frame_info_.size()) - 1, + next_frame = num_frames, + frame_shift = opts_.NccfWindowShift(), + next_frame_sample = frame_shift * next_frame; + + signal_sumsq_ += VecVec(downsampled_wave_part, downsampled_wave_part); + BaseFloat downsampled_wave_part_sum = 0.0; + for (int i = 0; i < downsampled_wave_part.size(); i++) + downsampled_wave_part_sum += downsampled_wave_part[i]; + signal_sum_ += downsampled_wave_part_sum; + + int next_downsampled_samples_processed = + downsampled_samples_processed_ + int(downsampled_wave_part.size()); + + if (next_frame_sample > next_downsampled_samples_processed) { + // this could only happen in the weird situation that the full frame length + // is less than the frame shift. + int full_frame_length = opts_.NccfWindowSize() + nccf_last_lag_; + assert(full_frame_length < frame_shift); + downsampled_signal_remainder_.resize(0); + } else { + vector new_remainder(next_downsampled_samples_processed - + next_frame_sample); + for (int i = next_frame_sample; + i < next_downsampled_samples_processed; i++) { + if (i >= downsampled_samples_processed_) { // in current signal. + new_remainder[i - next_frame_sample] = + downsampled_wave_part[i - downsampled_samples_processed_]; + } else { // in old remainder; only reach here if waveform supplied is + new_remainder[i - next_frame_sample] = // tiny. + downsampled_signal_remainder_[i - downsampled_samples_processed_ + + int(downsampled_signal_remainder_.size())]; + } + } + downsampled_signal_remainder_.swap(new_remainder); + } + downsampled_samples_processed_ = next_downsampled_samples_processed; +} + +void OnlinePitchFeatureImpl::ExtractFrame( + const vector &downsampled_wave_part, + int sample_index, + vector *window) { + int full_frame_length = window->size(); + int offset = static_cast(sample_index - + downsampled_samples_processed_); + // Treat edge cases first + if (sample_index < 0) { + assert(opts_.snip_edges == false); + int sub_frame_length = sample_index + full_frame_length; + int sub_frame_index = full_frame_length - sub_frame_length; + assert(sub_frame_length > 0 && sub_frame_index > 0); + for (int i = 0; i < window->size(); i++) + (*window)[i] = 0.0; + vector sub_window = sub_vector(*window, sub_frame_index, sub_frame_length); + ExtractFrame(downsampled_wave_part, 0, &sub_window); + return; + } + + if (offset + full_frame_length > int(downsampled_wave_part.size())) { + assert(input_finished_); + int sub_frame_length = int(downsampled_wave_part.size()) - offset; + assert(sub_frame_length > 0); + for (int i = 0; i < window->size(); i++) + (*window)[i] = 0.0; + vector sub_window = sub_vector(*window, 0, sub_frame_length); + ExtractFrame(downsampled_wave_part, sample_index, &sub_window); + return; + } + + if (offset >= 0) { + // frame is full inside the new part of the signal. + for (int k = 0; k < full_frame_length; k++){ + (*window)[k] = downsampled_wave_part[offset + k]; + } + } else { + // frame is partly in the remainder and partly in the new part. + int remainder_offset = int(downsampled_signal_remainder_.size()) + offset; + assert(remainder_offset >= 0); // or we didn't keep enough remainder. + assert(offset + full_frame_length > 0); // or we should have + // processed this frame last + // time. + + int old_length = -offset, new_length = offset + full_frame_length; + for (int i = 0; i < old_length; i++){ + (*window)[i] = downsampled_signal_remainder_[remainder_offset + i]; + } + for (int i = 0; i < new_length; i++){ + (*window)[i + old_length] = downsampled_wave_part[i]; + } + } + if (opts_.preemph_coeff != 0.0) { + BaseFloat preemph_coeff = opts_.preemph_coeff; + for (int i = int(window->size()) - 1; i > 0; i--) + (*window)[i] -= preemph_coeff * (*window)[i-1]; + (*window)[0] *= (1.0 - preemph_coeff); + } +} + +bool OnlinePitchFeatureImpl::IsLastFrame(int frame) const { + int T = NumFramesReady(); + assert(frame < T); + return (input_finished_ && frame + 1 == T); +} + +BaseFloat OnlinePitchFeatureImpl::FrameShiftInSeconds() const { + return opts_.frame_shift_ms / 1000.0f; +} + +int OnlinePitchFeatureImpl::NumFramesReady() const { + int num_frames = lag_nccf_.size(), + latency = frames_latency_; + assert(latency <= num_frames); + return num_frames - latency; +} + + +void OnlinePitchFeatureImpl::GetFrame(int frame, + vector *feat) { + assert(frame < NumFramesReady() && feat->size() == 2); + (*feat)[0] = lag_nccf_[frame].second; + (*feat)[1] = 1.0 / lags_[lag_nccf_[frame].first]; +} + +void OnlinePitchFeatureImpl::InputFinished() { + input_finished_ = true; + AcceptWaveform(opts_.samp_freq, vector()); + int num_frames = static_cast(int(frame_info_.size()) - 1); + if (num_frames < opts_.recompute_frame && !opts_.nccf_ballast_online) + RecomputeBacktraces(); + frames_latency_ = 0; +} + +void OnlinePitchFeatureImpl::RecomputeBacktraces() { + assert(!opts_.nccf_ballast_online); + int num_frames = static_cast(frame_info_.size()) - 1; + + // The assertion reflects how we believe this function will be called. + assert(num_frames <= opts_.recompute_frame); + assert(nccf_info_.size() == static_cast(num_frames)); + if (num_frames == 0) + return; + double num_samp = downsampled_samples_processed_, sum = signal_sum_, + sumsq = signal_sumsq_, mean = sum / num_samp; + BaseFloat mean_square = sumsq / num_samp - mean * mean; + + bool must_recompute = false; + BaseFloat threshold = 0.01; + for (int frame = 0; frame < num_frames; frame++) + if (!ApproxEqual(nccf_info_[frame]->mean_square_energy, + mean_square, threshold)) + must_recompute = true; + + if (!must_recompute) { + for (size_t i = 0; i < nccf_info_.size(); i++) + delete nccf_info_[i]; + nccf_info_.clear(); + return; + } + + int num_states = forward_cost_.size(), + basic_frame_length = opts_.NccfWindowSize(); + + BaseFloat new_nccf_ballast = pow(mean_square * basic_frame_length, 2) * + opts_.nccf_ballast; + + double forward_cost_remainder = 0.0; + vector forward_cost(num_states, 0), // start off at zero. + next_forward_cost(forward_cost); + std::vector > index_info; + + for (int frame = 0; frame < num_frames; frame++) { + NccfInfo &nccf_info = *nccf_info_[frame]; + BaseFloat old_mean_square = nccf_info_[frame]->mean_square_energy, + avg_norm_prod = nccf_info_[frame]->avg_norm_prod, + old_nccf_ballast = pow(old_mean_square * basic_frame_length, 2) * + opts_.nccf_ballast, + nccf_scale = pow((old_nccf_ballast + avg_norm_prod) / + (new_nccf_ballast + avg_norm_prod), + static_cast(0.5)); + for (int i = 0; i < nccf_info.nccf_pitch_resampled.size(); i++) + nccf_info.nccf_pitch_resampled[i] *= nccf_scale; + + frame_info_[frame + 1]->ComputeBacktraces( + opts_, nccf_info.nccf_pitch_resampled, lags_, + forward_cost, &index_info, &next_forward_cost); + + forward_cost.swap(next_forward_cost); + + BaseFloat remainder = vector_Min(forward_cost); + forward_cost_remainder += remainder; + for (int i = 0; i < forward_cost.size(); i++) + forward_cost[i] -= remainder; + } + + forward_cost_remainder_ = forward_cost_remainder; + forward_cost_.swap(forward_cost); + + int best_final_state; + BaseFloat forward_cost_min = vector_Min(forward_cost_); + int index_forward_cost_min = 0; + while (forward_cost_[index_forward_cost_min] != forward_cost_min) + index_forward_cost_min += 1; + best_final_state = index_forward_cost_min; + + if (lag_nccf_.size() != static_cast(num_frames)) + lag_nccf_.resize(num_frames); + + frame_info_.back()->SetBestState(best_final_state, lag_nccf_); + frames_latency_ = + frame_info_.back()->ComputeLatency(opts_.max_frames_latency); + for (size_t i = 0; i < nccf_info_.size(); i++) + delete nccf_info_[i]; + nccf_info_.clear(); +} + +OnlinePitchFeatureImpl::~OnlinePitchFeatureImpl() { + delete nccf_resampler_; + delete signal_resampler_; + for (size_t i = 0; i < frame_info_.size(); i++) + delete frame_info_[i]; + for (size_t i = 0; i < nccf_info_.size(); i++) + delete nccf_info_[i]; +} + +void OnlinePitchFeatureImpl::AcceptWaveform( + BaseFloat sampling_rate, + const vector &wave) { + // flush out the last few samples of input waveform only if input_finished_ == + // true. + const bool flush = input_finished_; + vector downsampled_wave; + signal_resampler_->Resample(wave, flush, &downsampled_wave); + double cur_sumsq = signal_sumsq_, cur_sum = signal_sum_; + int cur_num_samp = downsampled_samples_processed_, + prev_frame_end_sample = 0; + if (!opts_.nccf_ballast_online) { + cur_sumsq += VecVec(downsampled_wave, downsampled_wave); + BaseFloat downsampled_wave_sum = 0.0; + for (int j = 0; j < downsampled_wave.size(); j++) + downsampled_wave_sum += downsampled_wave[j]; + cur_num_samp += downsampled_wave.size(); + } + int end_frame = NumFramesAvailable( + downsampled_samples_processed_ + downsampled_wave.size(), opts_.snip_edges); + int start_frame = int(frame_info_.size()) - 1, + num_new_frames = end_frame - start_frame; + + if (num_new_frames == 0) { + UpdateRemainder(downsampled_wave); + return; + } + + int num_measured_lags = nccf_last_lag_ + 1 - nccf_first_lag_, + num_resampled_lags = lags_.size(), + frame_shift = opts_.NccfWindowShift(), + basic_frame_length = opts_.NccfWindowSize(), + full_frame_length = basic_frame_length + nccf_last_lag_; + + vector window(full_frame_length), + inner_prod(num_measured_lags), + norm_prod(num_measured_lags); + + vector> nccf_pitch(num_new_frames, vector(num_measured_lags)); + vector> nccf_pov(num_new_frames, vector(num_measured_lags)); + vector cur_forward_cost(num_resampled_lags); + + for (int frame = start_frame; frame < end_frame; frame++) { + // start_sample is index into the whole wave, not just this part. + int start_sample; + if (opts_.snip_edges) { + // Usual case: offset starts at 0 + start_sample = static_cast(frame) * frame_shift; + } else { + start_sample = + static_cast((frame + 0.5) * frame_shift) - full_frame_length / 2; + } + ExtractFrame(downsampled_wave, start_sample, &window); + if (opts_.nccf_ballast_online) { + int end_sample = start_sample + full_frame_length - + downsampled_samples_processed_; + assert(end_sample > 0); // or should have processed this frame last + // time. Note: end_sample is one past last + // sample. + if (end_sample > downsampled_wave.size()) { + assert(input_finished_); + end_sample = downsampled_wave.size(); + } + vector new_part = sub_vector(downsampled_wave, prev_frame_end_sample, + end_sample - prev_frame_end_sample); + cur_num_samp += int(new_part.size()); + cur_sumsq += VecVec(new_part, new_part); + BaseFloat new_part_sum = 0; + for (int k = 0; k < new_part.size(); k++) + new_part_sum += new_part[k]; + cur_sum += new_part_sum; + prev_frame_end_sample = end_sample; + } + double mean_square = cur_sumsq / cur_num_samp - + pow(cur_sum / cur_num_samp, 2.0); + ComputeCorrelation(window, nccf_first_lag_, nccf_last_lag_, + basic_frame_length, &inner_prod, &norm_prod); + BaseFloat norm_prod_sum = 0.0; + for (int k = 0; k < norm_prod.size(); k++) + norm_prod_sum += norm_prod[k]; + double nccf_ballast_pov = 0.0, + nccf_ballast_pitch = pow(mean_square * basic_frame_length, 2) * + opts_.nccf_ballast, + avg_norm_prod = norm_prod_sum / int(norm_prod.size()); + ComputeNccf(inner_prod, norm_prod, nccf_ballast_pitch, + &nccf_pitch[frame - start_frame]); + ComputeNccf(inner_prod, norm_prod, nccf_ballast_pov, + &nccf_pov[frame - start_frame]); + if (frame < opts_.recompute_frame) + nccf_info_.push_back(new NccfInfo(avg_norm_prod, mean_square)); + } + + vector> nccf_pitch_resampled(num_new_frames, vector(num_resampled_lags)); + nccf_resampler_->Resample(nccf_pitch, &nccf_pitch_resampled); + for (int i = 0; i < nccf_pitch.size(); i++) + vector().swap(nccf_pitch[i]); + nccf_pitch.clear(); + vector> nccf_pov_resampled(num_new_frames, vector(num_resampled_lags)); + nccf_resampler_->Resample(nccf_pov, &nccf_pov_resampled); + for (int i = 0; i < nccf_pov.size(); i++) + vector().swap(nccf_pov[i]); + nccf_pov.clear(); + UpdateRemainder(downsampled_wave); + + std::vector > index_info; + + for (int frame = start_frame; frame < end_frame; frame++) { + int frame_idx = frame - start_frame; + PitchFrameInfo *prev_info = frame_info_.back(), + *cur_info = new PitchFrameInfo(prev_info); + cur_info->SetNccfPov(nccf_pov_resampled[frame_idx]); + cur_info->ComputeBacktraces(opts_, nccf_pitch_resampled[frame_idx], + lags_, forward_cost_, &index_info, + &cur_forward_cost); + + forward_cost_.swap(cur_forward_cost); + BaseFloat remainder = vector_Min(forward_cost_); + forward_cost_remainder_ += remainder; + for (int k = 0; k < forward_cost_.size(); k++) + forward_cost_[k] -= remainder; + frame_info_.push_back(cur_info); + if (frame < opts_.recompute_frame) + nccf_info_[frame]->nccf_pitch_resampled = + nccf_pitch_resampled[frame_idx]; + if (frame == opts_.recompute_frame - 1 && !opts_.nccf_ballast_online) + RecomputeBacktraces(); + } + + int best_final_state; + BaseFloat forward_cost_min2 = vector_Min(forward_cost_); + for (int g = 0; g < forward_cost_.size(); g++){ + if (forward_cost_[g] == forward_cost_min2) + best_final_state = g; + } + lag_nccf_.resize(int(frame_info_.size()) - 1); // will keep any existing data. + frame_info_.back()->SetBestState(best_final_state, lag_nccf_); + frames_latency_ = + frame_info_.back()->ComputeLatency(opts_.max_frames_latency); +} + +int OnlinePitchFeature::NumFramesReady() const { + return impl_->NumFramesReady(); +} + +OnlinePitchFeature::OnlinePitchFeature(const PitchExtractionOptions &opts) + :impl_(new OnlinePitchFeatureImpl(opts)) { } + +bool OnlinePitchFeature::IsLastFrame(int frame) const { + return impl_->IsLastFrame(frame); +} + +BaseFloat OnlinePitchFeature::FrameShiftInSeconds() const { + return impl_->FrameShiftInSeconds(); +} + +void OnlinePitchFeature::GetFrame(int frame, vector *feat) { + impl_->GetFrame(frame, feat); +} + +void OnlinePitchFeature::AcceptWaveform( + BaseFloat sampling_rate, + const vector &waveform) { + impl_->AcceptWaveform(sampling_rate, waveform); +} + +void OnlinePitchFeature::InputFinished() { + impl_->InputFinished(); +} + +OnlinePitchFeature::~OnlinePitchFeature() { + delete impl_; +} + +void ComputeKaldiPitchFirstPass( + const PitchExtractionOptions &opts, + const vector &wave, + vector> *output) { + + int cur_rows = 100; + vector> feats(cur_rows, vector(2)); + + OnlinePitchFeature pitch_extractor(opts); + assert(opts.frames_per_chunk > 0 && + "--simulate-first-pass-online option does not make sense " + "unless you specify --frames-per-chunk"); + + int cur_offset = 0, cur_frame = 0, samp_per_chunk = + opts.frames_per_chunk * opts.samp_freq * opts.frame_shift_ms / 1000.0f; + + while (cur_offset < wave.size()) { + int num_samp = std::min(samp_per_chunk, int(wave.size()) - cur_offset); + vector wave_chunk = sub_vector(wave, cur_offset, num_samp); + pitch_extractor.AcceptWaveform(opts.samp_freq, wave_chunk); + cur_offset += num_samp; + if (cur_offset == wave.size()) + pitch_extractor.InputFinished(); + // Get each frame as soon as it is ready. + for (; cur_frame < pitch_extractor.NumFramesReady(); cur_frame++) { + if (cur_frame >= cur_rows) { + cur_rows *= 2; + feats.resize(cur_rows); + for(int i = 0; i < cur_rows; i++) + feats[i].resize(2); + } + vector row(feats[cur_frame]); + pitch_extractor.GetFrame(cur_frame, &row); + } + } + if (cur_frame == 0) { + std::cerr << "No features output since wave file too short"; + for (int i = 0; i < output->size(); i++) + vector().swap((*output)[i]); + (*output).clear(); + } else { + for(int j = 0; j < cur_frame; j++) + (*output)[j] = feats[j]; + } +} + +void ComputeKaldiPitch(const PitchExtractionOptions &opts, + const vector &wave, + vector> *output) { + if (opts.simulate_first_pass_online) { + ComputeKaldiPitchFirstPass(opts, wave, output); + return; + } + OnlinePitchFeature pitch_extractor(opts); + + if (opts.frames_per_chunk == 0) { + pitch_extractor.AcceptWaveform(opts.samp_freq, wave); + } else { + // the user may set opts.frames_per_chunk for better compatibility with + // online operation. + assert(opts.frames_per_chunk > 0); + int cur_offset = 0, samp_per_chunk = + opts.frames_per_chunk * opts.samp_freq * opts.frame_shift_ms / 1000.0f; + while (cur_offset < wave.size()) { + int num_samp = std::min(samp_per_chunk, int(wave.size()) - cur_offset); + vector wave_chunk = sub_vector(wave, cur_offset, num_samp); + pitch_extractor.AcceptWaveform(opts.samp_freq, wave_chunk); + cur_offset += num_samp; + } + } + + pitch_extractor.InputFinished(); + int num_frames = pitch_extractor.NumFramesReady(); + if (num_frames == 0) { + std::cerr << "No frames output in pitch extraction"; + for (int i = 0; i < output->size(); i++) + vector().swap((*output)[i]); + (*output).clear(); + return; + } + + output->resize(num_frames); + for(int i = 0; i < num_frames; i++) + (*output)[i].resize(2); + for (int frame = 0; frame < num_frames; frame++) { + pitch_extractor.GetFrame(frame, &((*output)[frame])); + } +} + +} // namespace delta diff --git a/athena/transform/feats/ops/kernels/pitch.h b/athena/transform/feats/ops/kernels/pitch.h new file mode 100644 index 00000000..c6fcff92 --- /dev/null +++ b/athena/transform/feats/ops/kernels/pitch.h @@ -0,0 +1,187 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef DELTA_LAYERS_OPS_KERNELS_PITCH_H_ +#define DELTA_LAYERS_OPS_KERNELS_PITCH_H_ + +#include +#include +#include +#include +#include + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/platform/logging.h" +#include "kernels/support_functions.h" +#include "kernels/resample.h" + +using namespace tensorflow; // NOLINT + +namespace delta { + +struct PitchExtractionOptions { + // FrameExtractionOptions frame_opts; + BaseFloat samp_freq; // sample frequency in hertz + BaseFloat frame_shift_ms; // in milliseconds. + BaseFloat frame_length_ms; // in milliseconds. + BaseFloat preemph_coeff; // Preemphasis coefficient. [use is deprecated.] + BaseFloat min_f0; // min f0 to search (Hz) + BaseFloat max_f0; // max f0 to search (Hz) + BaseFloat soft_min_f0; // Minimum f0, applied in soft way, must not + // exceed min-f0 + BaseFloat penalty_factor; // cost factor for FO change + BaseFloat lowpass_cutoff; // cutoff frequency for Low pass filter + BaseFloat resample_freq; // Integer that determines filter width when + // upsampling NCCF + BaseFloat delta_pitch; // the pitch tolerance in pruning lags + BaseFloat nccf_ballast; // Increasing this factor reduces NCCF for + // quiet frames, helping ensure pitch + // continuity in unvoiced region + int lowpass_filter_width; // Integer that determines filter width of + // lowpass filter + int upsample_filter_width; // Integer that determines filter width when + // upsampling NCCF + + int max_frames_latency; + int frames_per_chunk; + + bool simulate_first_pass_online; + + int recompute_frame; + + bool nccf_ballast_online; + bool snip_edges; + PitchExtractionOptions(): + samp_freq(16000), + frame_shift_ms(10.0), + frame_length_ms(25.0), + preemph_coeff(0.0), + min_f0(50), + max_f0(400), + soft_min_f0(10.0), + penalty_factor(0.1), + lowpass_cutoff(1000), + resample_freq(4000), + delta_pitch(0.005), + nccf_ballast(7000), + lowpass_filter_width(1), + upsample_filter_width(5), + max_frames_latency(0), + frames_per_chunk(0), + simulate_first_pass_online(false), + recompute_frame(500), + nccf_ballast_online(false), + snip_edges(true) { } + + void set_samp_freq(BaseFloat sample_rate) { samp_freq = sample_rate; } + void set_frame_shift_ms(BaseFloat frame_shift) { frame_shift_ms = frame_shift * 1000; } + void set_frame_length_ms(BaseFloat window_length) { frame_length_ms = window_length * 1000; } + void set_preemph_coeff(BaseFloat preemph_coeff_) { preemph_coeff = preemph_coeff_; } + void set_min_f0(BaseFloat min_f0_) { min_f0 = min_f0_; } + void set_max_f0(BaseFloat max_f0_) { max_f0 = max_f0_; } + void set_soft_min_f0(BaseFloat soft_min_f0_) { soft_min_f0 = soft_min_f0_; } + void set_penalty_factor(BaseFloat penalty_factor_) { penalty_factor = penalty_factor_; } + void set_lowpass_cutoff(BaseFloat lowpass_cutoff_) { lowpass_cutoff = lowpass_cutoff_; } + void set_resample_freq(BaseFloat resample_freq_) { resample_freq = resample_freq_; } + void set_delta_pitch(BaseFloat delta_pitch_) { delta_pitch = delta_pitch_; } + void set_nccf_ballast(BaseFloat nccf_ballast_) { nccf_ballast = nccf_ballast_; } + void set_lowpass_filter_width(int lowpass_filter_width_) { lowpass_filter_width = lowpass_filter_width_; } + void set_upsample_filter_width(int upsample_filter_width_) { upsample_filter_width = upsample_filter_width_; } + void set_max_frames_latency(int max_frames_latency_) { max_frames_latency = max_frames_latency_; } + void set_frames_per_chunk(int frames_per_chunk_) { frames_per_chunk = frames_per_chunk_; } + void set_simulate_first_pass_online(bool simulate_first_pass_online_) { simulate_first_pass_online = simulate_first_pass_online_; } + void set_recompute_frame(int recompute_frame_) { recompute_frame = recompute_frame_; } + void set_nccf_ballast_online(bool nccf_ballast_online_) { nccf_ballast_online = nccf_ballast_online_; } + void set_snip_edges(bool snip_edges_) { snip_edges = snip_edges_; } + + int NccfWindowSize() const { + return static_cast(resample_freq * frame_length_ms / 1000.0); + } + /// Returns the window-shift in samples, after resampling. + int NccfWindowShift() const { + return static_cast(resample_freq * frame_shift_ms / 1000.0); + } +}; + +class OnlinePitchFeatureImpl; + +class OnlineFeatureInterface { + public: + virtual int Dim() const = 0; /// returns the feature dimension. + virtual int NumFramesReady() const = 0; + + virtual bool IsLastFrame(int frame) const = 0; + virtual void GetFrame(int frame, vector *feat) = 0; + + virtual void GetFrames(const std::vector &frames, + vector > *feats) { + for (int i = 0; i < frames.size(); i++) { + vector feat = (*feats)[i]; + GetFrame(frames[i], &feat); + } + } + + virtual BaseFloat FrameShiftInSeconds() const = 0; + virtual ~OnlineFeatureInterface() { } + +}; + +class OnlineBaseFeature: public OnlineFeatureInterface { + public: + virtual void AcceptWaveform(BaseFloat sampling_rate, + const vector &waveform) = 0; + + virtual void InputFinished() = 0; +}; + + + +// Note: to start on a new waveform, just construct a new version +// of this object. +class OnlinePitchFeature: public OnlineBaseFeature { + public: + explicit OnlinePitchFeature(const PitchExtractionOptions &opts); + + virtual int Dim() const { return 2; /* (NCCF, pitch) */ } + + virtual int NumFramesReady() const; + + virtual BaseFloat FrameShiftInSeconds() const; + + virtual bool IsLastFrame(int frame) const; + + /// Outputs the two-dimensional feature consisting of (pitch, NCCF). You + /// should probably post-process this using class OnlineProcessPitch. + virtual void GetFrame(int frame, vector *feat); + + virtual void AcceptWaveform(BaseFloat sampling_rate, + const vector &waveform); + + virtual void InputFinished(); + + virtual ~OnlinePitchFeature(); + + private: + OnlinePitchFeatureImpl *impl_; +}; + +/// This function extracts (pitch, NCCF) per frame. +void ComputeKaldiPitch(const PitchExtractionOptions &opts, + const vector &wave, + vector> *output); + +} // namespace delta +#endif // DELTA_LAYERS_OPS_KERNELS_PITCH_H_ diff --git a/athena/transform/feats/ops/kernels/pitch_op.cc b/athena/transform/feats/ops/kernels/pitch_op.cc new file mode 100644 index 00000000..fbd8c5c1 --- /dev/null +++ b/athena/transform/feats/ops/kernels/pitch_op.cc @@ -0,0 +1,152 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "kernels/pitch.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/status.h" + +namespace delta { + +class PitchOp : public OpKernel { + public: + explicit PitchOp(OpKernelConstruction* context) : OpKernel(context) { + OP_REQUIRES_OK(context, + context->GetAttr("snip_edges", &snip_edges_)); + OP_REQUIRES_OK(context, + context->GetAttr("window_length", &window_length_)); + OP_REQUIRES_OK(context, + context->GetAttr("frame_length", &frame_length_)); + OP_REQUIRES_OK(context, + context->GetAttr("preemph_coeff", &preemph_coeff_)); + OP_REQUIRES_OK(context, + context->GetAttr("min_f0", &min_f0_)); + OP_REQUIRES_OK(context, + context->GetAttr("max_f0", &max_f0_)); + OP_REQUIRES_OK(context, + context->GetAttr("soft_min_f0", &soft_min_f0_)); + OP_REQUIRES_OK(context, + context->GetAttr("penalty_factor", &penalty_factor_)); + OP_REQUIRES_OK(context, + context->GetAttr("lowpass_cutoff", &lowpass_cutoff_)); + OP_REQUIRES_OK(context, + context->GetAttr("resample_freq", &resample_freq_)); + OP_REQUIRES_OK(context, + context->GetAttr("delta_pitch", &delta_pitch_)); + OP_REQUIRES_OK(context, + context->GetAttr("nccf_ballast", &nccf_ballast_)); + OP_REQUIRES_OK(context, + context->GetAttr("lowpass_filter_width", &lowpass_filter_width_)); + OP_REQUIRES_OK(context, + context->GetAttr("upsample_filter_width", &upsample_filter_width_)); + OP_REQUIRES_OK(context, + context->GetAttr("max_frames_latency", &max_frames_latency_)); + OP_REQUIRES_OK(context, + context->GetAttr("frames_per_chunk", &frames_per_chunk_)); + OP_REQUIRES_OK(context, + context->GetAttr("simulate_first_pass_online", &simulate_first_pass_online_)); + OP_REQUIRES_OK(context, + context->GetAttr("recompute_frame", &recompute_frame_)); + OP_REQUIRES_OK(context, + context->GetAttr("nccf_ballast_online", &nccf_ballast_online_)); + } + + void Compute(OpKernelContext* context) override { + const Tensor& input_tensor = context->input(0); + OP_REQUIRES(context, input_tensor.dims() == 1, + errors::InvalidArgument("input signal must be 1-dimensional", + input_tensor.shape().DebugString())); + const Tensor& sample_rate_tensor = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(sample_rate_tensor.shape()), + errors::InvalidArgument( + "Input sample_rate should be a scalar tensor, got ", + sample_rate_tensor.shape().DebugString(), " instead.")); + const int32 sample_rate = sample_rate_tensor.scalar()(); + const float* input_flat = input_tensor.flat().data(); + + Tensor* output_tensor = nullptr; + const int L = input_tensor.dim_size(0); + int i_WinLen = static_cast(window_length_ * sample_rate); + int i_FrmLen = static_cast(frame_length_ * sample_rate); + int i_NumFrm = (L - i_WinLen) / i_FrmLen + 1; + if (snip_edges_ == false) + i_NumFrm = (L + i_FrmLen / 2) / i_FrmLen; + OP_REQUIRES_OK( + context, context->allocate_output(0, TensorShape({i_NumFrm, 2}), + &output_tensor)); + float* output_flat = output_tensor->flat().data(); + PitchExtractionOptions pitch_opts; + pitch_opts.set_samp_freq(static_cast(sample_rate)); + pitch_opts.set_frame_shift_ms(static_cast(frame_length_)); + pitch_opts.set_frame_length_ms(static_cast(window_length_)); + pitch_opts.set_preemph_coeff(static_cast(preemph_coeff_)); + pitch_opts.set_min_f0(static_cast(min_f0_)); + pitch_opts.set_max_f0(static_cast(max_f0_)); + pitch_opts.set_soft_min_f0(static_cast(soft_min_f0_)); + pitch_opts.set_penalty_factor(static_cast(penalty_factor_)); + pitch_opts.set_lowpass_cutoff(static_cast(lowpass_cutoff_)); + pitch_opts.set_resample_freq(static_cast(resample_freq_)); + pitch_opts.set_delta_pitch(static_cast(delta_pitch_)); + pitch_opts.set_nccf_ballast(static_cast(nccf_ballast_)); + pitch_opts.set_lowpass_filter_width(lowpass_filter_width_); + pitch_opts.set_upsample_filter_width(upsample_filter_width_); + pitch_opts.set_max_frames_latency(max_frames_latency_); + pitch_opts.set_frames_per_chunk(frames_per_chunk_); + pitch_opts.set_simulate_first_pass_online(simulate_first_pass_online_); + pitch_opts.set_recompute_frame(recompute_frame_); + pitch_opts.set_nccf_ballast_online(nccf_ballast_online_); + pitch_opts.set_snip_edges(snip_edges_); + vector> features(i_NumFrm, vector(2)); + vector waveform(L); + for (int i = 0; i < L; i++){ + waveform[i] = static_cast(input_flat[i]); + } + ComputeKaldiPitch(pitch_opts, waveform, &features); + for(int j = 0; j < i_NumFrm; j++){ + for(int k = 0; k < 2; k++){ + output_flat[j * 2 + k] = static_cast(features[j][k]); + } + } + } + + private: + float window_length_; + float frame_length_; + float preemph_coeff_; + float min_f0_; + float max_f0_; + float soft_min_f0_; + float penalty_factor_; + float lowpass_cutoff_; + float resample_freq_; + float delta_pitch_; + float nccf_ballast_; + int lowpass_filter_width_; + int upsample_filter_width_; + int max_frames_latency_; + int frames_per_chunk_; + bool simulate_first_pass_online_; + int recompute_frame_; + bool nccf_ballast_online_; + bool snip_edges_; +}; + +REGISTER_KERNEL_BUILDER(Name("Pitch").Device(DEVICE_CPU), PitchOp); + +} // namespace delta \ No newline at end of file diff --git a/athena/transform/feats/ops/kernels/resample.cc b/athena/transform/feats/ops/kernels/resample.cc new file mode 100644 index 00000000..ed6d1be5 --- /dev/null +++ b/athena/transform/feats/ops/kernels/resample.cc @@ -0,0 +1,313 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include "kernels/resample.h" +#include "kernels/support_functions.h" + +using namespace std; +namespace delta { + +LinearResample::LinearResample(int samp_rate_in_hz, + int samp_rate_out_hz, + BaseFloat filter_cutoff_hz, + int num_zeros): + samp_rate_in_(samp_rate_in_hz), + samp_rate_out_(samp_rate_out_hz), + filter_cutoff_(filter_cutoff_hz), + num_zeros_(num_zeros) { + assert(samp_rate_in_hz > 0.0 && + samp_rate_out_hz > 0.0 && + filter_cutoff_hz > 0.0 && + filter_cutoff_hz*2 <= samp_rate_out_hz && + num_zeros > 0); + + // base_freq is the frequency of the repeating unit, which is the gcd + // of the input frequencies. + int base_freq = Gcd(samp_rate_in_, samp_rate_out_); + input_samples_in_unit_ = samp_rate_in_ / base_freq; + output_samples_in_unit_ = samp_rate_out_ / base_freq; + + SetIndexesAndWeights(); + Reset(); +} + +int LinearResample::GetNumOutputSamples(int input_num_samp, + bool flush) const { + int tick_freq = Lcm(samp_rate_in_, samp_rate_out_); + int ticks_per_input_period = tick_freq / samp_rate_in_; + + // work out the number of ticks in the time interval + // [ 0, input_num_samp/samp_rate_in_ ). + long long interval_length_in_ticks = (long long)input_num_samp * (long long)ticks_per_input_period; + if (!flush) { + BaseFloat window_width = num_zeros_ / (2.0 * filter_cutoff_); + int window_width_ticks = floor(window_width * tick_freq); + interval_length_in_ticks -= window_width_ticks; + } + if (interval_length_in_ticks <= 0) + return 0; + int ticks_per_output_period = tick_freq / samp_rate_out_; + int last_output_samp = interval_length_in_ticks / ticks_per_output_period; + if (last_output_samp * ticks_per_output_period == interval_length_in_ticks) + last_output_samp--; + int num_output_samp = last_output_samp + 1; + return num_output_samp; +} + +void LinearResample::SetIndexesAndWeights() { + first_index_.resize(output_samples_in_unit_); + weights_.resize(output_samples_in_unit_); + + double window_width = num_zeros_ / (2.0 * filter_cutoff_); + + for (int i = 0; i < output_samples_in_unit_; i++) { + double output_t = i / static_cast(samp_rate_out_); + double min_t = output_t - window_width, max_t = output_t + window_width; + int min_input_index = ceil(min_t * samp_rate_in_), + max_input_index = floor(max_t * samp_rate_in_), + num_indices = max_input_index - min_input_index + 1; + first_index_[i] = min_input_index; + weights_[i].resize(num_indices); + for (int j = 0; j < num_indices; j++) { + int input_index = min_input_index + j; + double input_t = input_index / static_cast(samp_rate_in_), + delta_t = input_t - output_t; + // sign of delta_t doesn't matter. + weights_[i][j] = FilterFunc(delta_t) / samp_rate_in_; + } + } +} + + +// inline +void LinearResample::GetIndexes(int samp_out, + int *first_samp_in, + int *samp_out_wrapped) const { + int unit_index = samp_out / output_samples_in_unit_; + // samp_out_wrapped is equal to samp_out % output_samples_in_unit_ + *samp_out_wrapped = static_cast(samp_out - + unit_index * output_samples_in_unit_); + *first_samp_in = first_index_[*samp_out_wrapped] + + unit_index * input_samples_in_unit_; +} + + +void LinearResample::Resample(const vector &input, + bool flush, + vector *output) { + int input_dim = input.size(); + int tot_input_samp = input_sample_offset_ + input_dim, + tot_output_samp = GetNumOutputSamples(tot_input_samp, flush); + + assert(tot_output_samp >= output_sample_offset_); + + output->resize(tot_output_samp - output_sample_offset_); + + // samp_out is the index into the total output signal, not just the part + // of it we are producing here. + for (int samp_out = output_sample_offset_; + samp_out < tot_output_samp; + samp_out++) { + int first_samp_in; + int samp_out_wrapped; + GetIndexes(samp_out, &first_samp_in, &samp_out_wrapped); + const vector &weights = weights_[samp_out_wrapped]; + int first_input_index = static_cast(first_samp_in - + input_sample_offset_); + BaseFloat this_output; + if (first_input_index >= 0 && + first_input_index + int(weights.size()) <= input_dim) { + vector input_part = sub_vector(input, first_input_index, int(weights.size())); + this_output = VecVec(input_part, weights); + } else { // Handle edge cases. + this_output = 0.0; + for (int i = 0; i < weights.size(); i++) { + BaseFloat weight = weights[i]; + int input_index = first_input_index + i; + if ((input_index < 0) && (int(input_remainder_.size()) + input_index >= 0)) { + this_output += weight * input_remainder_[int(input_remainder_.size()) + input_index]; + } + else if (input_index >= 0 && input_index < input_dim) { + this_output += weight * input[input_index]; + } + else if (input_index >= input_dim) { + assert(flush); + } + } + } + int output_index = static_cast(samp_out - output_sample_offset_); + (*output)[output_index] = this_output; + } + + if (flush) { + Reset(); // Reset the internal state. + } else { + SetRemainder(input); + input_sample_offset_ = tot_input_samp; + output_sample_offset_ = tot_output_samp; + } +} + +void LinearResample::SetRemainder(const vector &input) { + vector old_remainder(input_remainder_); + int max_remainder_needed = ceil(samp_rate_in_ * num_zeros_ / + filter_cutoff_); + input_remainder_.resize(max_remainder_needed); + for (int index = - input_remainder_.size(); index < 0; index++) { + int input_index = index + input.size(); + if (input_index >= 0) + input_remainder_[index + input_remainder_.size()] = input[input_index]; + else if (input_index + old_remainder.size() >= 0) + input_remainder_[index + input_remainder_.size()] = + old_remainder[input_index + old_remainder.size()]; + // else leave it at zero. + } +} + +void LinearResample::Reset() { + input_sample_offset_ = 0; + output_sample_offset_ = 0; + input_remainder_.resize(0); +} + +BaseFloat LinearResample::FilterFunc(BaseFloat t) const { + BaseFloat window, // raised-cosine (Hanning) window of width + // num_zeros_/2*filter_cutoff_ + filter; // sinc filter function + if (fabs(t) < num_zeros_ / (2.0 * filter_cutoff_)) + window = 0.5 * (1 + cos(M_2PI * filter_cutoff_ / num_zeros_ * t)); + else + window = 0.0; // outside support of window function + if (t != 0) + filter = sin(M_2PI * filter_cutoff_ * t) / (M_PI * t); + else + filter = 2 * filter_cutoff_; // limit of the function at t = 0 + return filter * window; +} + + +ArbitraryResample::ArbitraryResample( + int num_samples_in, BaseFloat samp_rate_in, + BaseFloat filter_cutoff, const vector &sample_points, + int num_zeros): + num_samples_in_(num_samples_in), + samp_rate_in_(samp_rate_in), + filter_cutoff_(filter_cutoff), + num_zeros_(num_zeros) { + assert(num_samples_in > 0 && samp_rate_in > 0.0 && + filter_cutoff > 0.0 && + filter_cutoff * 2.0 <= samp_rate_in + && num_zeros > 0); + // set up weights_ and indices_. Please try to keep all functions short and + SetIndexes(sample_points); + SetWeights(sample_points); +} + +void ArbitraryResample::Resample(const std::vector > &input, + std::vector > *output) const { + // each row of "input" corresponds to the data to resample; + // the corresponding row of "output" is the resampled data. + + vector output_col(output->size()); + int col_num = (*output)[0].size(); + for (int i = 0; i < NumSamplesOut(); i++) { + vector> input_part = sub_matrix(input, 0, input.size(), + first_index_[i], weights_[i].size()); + const vector &weight_vec(weights_[i]); + output_col = add_mat_vec(1.0, input_part, weight_vec, 0.0); + for (int j = 0; j < output_col.size(); j++){ + (*output)[j][i] = output_col[j]; + } + } +} + + +void ArbitraryResample::Resample(const vector &input, + vector *output) const { + assert(input.size() == num_samples_in_ && + output->size() == weights_.size()); + + int output_dim = output->size(); + for (int i = 0; i < output_dim; i++) { + vector input_part = sub_vector(input, first_index_[i], weights_[i].size()); + (*output)[i] = VecVec(input_part, weights_[i]); + } +} + +void ArbitraryResample::SetIndexes(const vector &sample_points) { + int num_samples = sample_points.size(); + first_index_.resize(num_samples); + weights_.resize(num_samples); + BaseFloat filter_width = num_zeros_ / (2.0 * filter_cutoff_); + for (int i = 0; i < num_samples; i++) { + // the t values are in seconds. + BaseFloat t = sample_points[i], + t_min = t - filter_width, t_max = t + filter_width; + int index_min = ceil(samp_rate_in_ * t_min), + index_max = floor(samp_rate_in_ * t_max); + // the ceil on index min and the floor on index_max are because there + // is no point using indices just outside the window (coeffs would be zero). + if (index_min < 0) + index_min = 0; + if (index_max >= num_samples_in_) + index_max = num_samples_in_ - 1; + first_index_[i] = index_min; + weights_[i].resize(index_max - index_min + 1); + } +} + +void ArbitraryResample::SetWeights(const vector &sample_points) { + int num_samples_out = NumSamplesOut(); + for (int i = 0; i < num_samples_out; i++) { + for (int j = 0 ; j < weights_[i].size(); j++) { + BaseFloat delta_t = sample_points[i] - + (first_index_[i] + j) / samp_rate_in_; + // Include at this point the factor of 1.0 / samp_rate_in_ which + // appears in the math. + weights_[i][j] = FilterFunc(delta_t) / samp_rate_in_; + } + } +} + +BaseFloat ArbitraryResample::FilterFunc(BaseFloat t) const { + BaseFloat window, // raised-cosine (Hanning) window of width + // num_zeros_/2*filter_cutoff_ + filter; // sinc filter function + if (fabs(t) < num_zeros_ / (2.0 * filter_cutoff_)) + window = 0.5 * (1 + cos(M_2PI * filter_cutoff_ / num_zeros_ * t)); + else + window = 0.0; // outside support of window function + if (t != 0.0) + filter = sin(M_2PI * filter_cutoff_ * t) / (M_PI * t); + else + filter = 2.0 * filter_cutoff_; // limit of the function at zero. + return filter * window; +} + +void ResampleWaveform(BaseFloat orig_freq, const vector &wave, + BaseFloat new_freq, vector *new_wave) { + BaseFloat min_freq = std::min(orig_freq, new_freq); + BaseFloat lowpass_cutoff = 0.99 * 0.5 * min_freq; + int lowpass_filter_width = 6; + LinearResample resampler(orig_freq, new_freq, + lowpass_cutoff, lowpass_filter_width); + resampler.Resample(wave, true, new_wave); +} +} // namespace delta diff --git a/athena/transform/feats/ops/kernels/resample.h b/athena/transform/feats/ops/kernels/resample.h new file mode 100644 index 00000000..653ea16f --- /dev/null +++ b/athena/transform/feats/ops/kernels/resample.h @@ -0,0 +1,123 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + + +#ifndef DELTA_LAYERS_OPS_KERNELS_RESAMPLE_H_ +#define DELTA_LAYERS_OPS_KERNELS_RESAMPLE_H_ + +#include +#include +#include +#include +#include +#include +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/platform/logging.h" +#include "kernels/support_functions.h" + +using namespace tensorflow; // NOLINT +using namespace std; + +namespace delta { + +class ArbitraryResample { + public: + ArbitraryResample(int num_samples_in, + BaseFloat samp_rate_hz, + BaseFloat filter_cutoff_hz, + const vector &sample_points_secs, + int num_zeros); + + int NumSamplesIn() const { return num_samples_in_; } + + int NumSamplesOut() const { return weights_.size(); } + void Resample(const std::vector > &input, + std::vector > *output) const; + + void Resample(const vector &input, + vector *output) const; + private: + void SetIndexes(const vector &sample_points); + + void SetWeights(const vector &sample_points); + + BaseFloat FilterFunc(BaseFloat t) const; + + int num_samples_in_; + BaseFloat samp_rate_in_; + BaseFloat filter_cutoff_; + int num_zeros_; + + std::vector first_index_; + std::vector > weights_; +}; + +class LinearResample { + public: + LinearResample(int samp_rate_in_hz, + int samp_rate_out_hz, + BaseFloat filter_cutoff_hz, + int num_zeros); + + void Resample(const vector &input, + bool flush, + vector *output); + + void Reset(); + + //// Return the input and output sampling rates (for checks, for example) + inline int GetInputSamplingRate() { return samp_rate_in_; } + inline int GetOutputSamplingRate() { return samp_rate_out_; } + private: + int GetNumOutputSamples(int input_num_samp, bool flush) const; + + inline void GetIndexes(int samp_out, + int *first_samp_in, + int *samp_out_wrapped) const; + + void SetRemainder(const vector &input); + + void SetIndexesAndWeights(); + + BaseFloat FilterFunc(BaseFloat) const; + + // The following variables are provided by the user. + int samp_rate_in_; + int samp_rate_out_; + BaseFloat filter_cutoff_; + int num_zeros_; + + int input_samples_in_unit_; + int output_samples_in_unit_; + + std::vector first_index_; + std::vector > weights_; + + int input_sample_offset_; + int output_sample_offset_; + vector input_remainder_; +}; + +void ResampleWaveform(BaseFloat orig_freq, const vector &wave, + BaseFloat new_freq, vector *new_wave); + +inline void DownsampleWaveForm(BaseFloat orig_freq, const vector &wave, + BaseFloat new_freq, vector *new_wave) { + ResampleWaveform(orig_freq, wave, new_freq, new_wave); +} + +} // namespace delta +#endif // DELTA_LAYERS_OPS_KERNELS_RESAMPLE_H_ diff --git a/athena/transform/feats/ops/kernels/spectrum.cc b/athena/transform/feats/ops/kernels/spectrum.cc new file mode 100644 index 00000000..eec21b11 --- /dev/null +++ b/athena/transform/feats/ops/kernels/spectrum.cc @@ -0,0 +1,210 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "kernels/spectrum.h" +#include "kernels/support_functions.h" + +namespace delta { +const float window_length_sec = 0.025; +const float frame_length_sec = 0.010; + +Spectrum::Spectrum() { + window_length_sec_ = window_length_sec; + frame_length_sec_ = frame_length_sec; + i_OutTyp = 1; + i_snip_edges = 1; + i_raw_energy = 1; + f_PreEph = 0.97; + i_is_fbank = true; + i_remove_dc_offset = true; + i_dither = 0.0; + snprintf(s_WinTyp, sizeof(s_WinTyp), "povey"); + pf_WINDOW = NULL; + pf_SPC = NULL; + win_temp = NULL; + win_buf = NULL; + eph_buf = NULL; + win = NULL; + fftwin = NULL; + fft_buf = NULL; +} + +void Spectrum::set_window_length_sec(float window_length_sec) { + window_length_sec_ = window_length_sec; +} + +void Spectrum::set_frame_length_sec(float frame_length_sec) { + frame_length_sec_ = frame_length_sec; +} + +void Spectrum::set_output_type(int output_type) { i_OutTyp = output_type; } + +void Spectrum::set_snip_edges(int snip_edges) { i_snip_edges = snip_edges; } + +void Spectrum::set_raw_energy(int raw_energy) {i_raw_energy = raw_energy;} + +void Spectrum::set_is_fbank(bool is_fbank) {i_is_fbank = is_fbank;} + +void Spectrum::set_remove_dc_offset(bool remove_dc_offset) {i_remove_dc_offset = remove_dc_offset;} + +void Spectrum::set_preEph(float preEph) {f_PreEph = preEph;} + +void Spectrum::set_dither(float dither) {i_dither = dither;} + +void Spectrum::set_window_type(char* window_type){ + snprintf(s_WinTyp, sizeof(s_WinTyp), "%s", window_type); +} + +int Spectrum::init_spc(int input_size, float sample_rate) { + f_SamRat = sample_rate; + i_WinLen = static_cast(window_length_sec_ * f_SamRat); + i_FrmLen = static_cast(frame_length_sec_ * f_SamRat); + if (i_snip_edges == 1) + i_NumFrm = (input_size - i_WinLen) / i_FrmLen + 1; + else + i_NumFrm = (input_size + i_FrmLen / 2) / i_FrmLen; + if (i_NumFrm < 1) + i_NumFrm = 1; + i_FFTSiz = static_cast(pow(2.0f, ceil(log2(i_WinLen)))); + i_NumFrq = i_FFTSiz / 2 + 1; + + return 1; +} + +int Spectrum::proc_spc(const float* mic_buf, int input_size) { + int n, k; + + if (input_size < i_WinLen) + std::cerr<<"Wraning: The length of input data is shorter than "<< window_length_sec_ << " s." <(malloc(sizeof(float) * i_WinLen)); + pf_SPC = static_cast(malloc(sizeof(float) * i_NumFrq * i_NumFrm)); + win = static_cast(malloc(sizeof(xcomplex) * i_FFTSiz)); + win_buf = static_cast(malloc(sizeof(float) * i_WinLen)); + eph_buf = static_cast(malloc(sizeof(float) * i_WinLen)); + win_temp = static_cast(malloc(sizeof(float) * i_WinLen)); + fftwin = static_cast(malloc(sizeof(xcomplex) * i_FFTSiz)); + fft_buf = static_cast(malloc(sizeof(float) * 2 * i_FFTSiz)); // c.r&c.i + + /* generate window */ + gen_window(pf_WINDOW, i_WinLen, s_WinTyp); + + for (n = 0; n < i_NumFrm; n++) { + float signal_raw_log_energy = 0.0; + float sum = 0.0; + for (int l = 0; l < i_WinLen; l++){ + int index = n * i_FrmLen + l; + if (index < input_size) { + win_buf[l] = mic_buf[index]; + } else { + win_buf[l] = 0.0f; + } + sum += win_buf[l]; + } + + if(i_dither != 0.0) { + do_dither(win_buf, i_WinLen, i_dither); + } + + if (i_remove_dc_offset == true){ + float mean = sum / i_WinLen; + for (int l = 0; l < i_WinLen; l++) { + win_buf[l] -= mean; + } + } + + /* do pre-emphais */ + do_frame_preemphasis(win_buf, eph_buf, i_WinLen, f_PreEph); + + for (k = 0; k < i_WinLen; k++) { + win[k].r = eph_buf[k] * pf_WINDOW[k]; + win[k].i = 0.0f; + } + + if (i_raw_energy == 1) { + std::memcpy(win_temp, win_buf, i_WinLen * sizeof(float)); + } + else { + for (k = 0; k < i_WinLen; k++) { + win_temp[k] = win[k].r; + } + } + + std::memset((void*)&(win[i_WinLen]), 0, sizeof(float) * 2 * (i_FFTSiz - i_WinLen));; + + /* raw energy */ + signal_raw_log_energy = compute_energy(win_temp, i_WinLen); + + /* fft */ + dit_r2_fft(win, fftwin, fft_buf, i_FFTSiz, -1); + + if (!i_is_fbank) { + fftwin[0].r = sqrt(signal_raw_log_energy); + fftwin[0].i = 0.0f; + } + + if (i_OutTyp == 1) { + for (k = 0; k < i_NumFrq; k++) { + pf_SPC[n * i_NumFrq + k] = complex_abs2(fftwin[k]); + } + } else if (i_OutTyp == 2) { + for (k = 0; k < i_NumFrq; k++) { + pf_SPC[n * i_NumFrq + k] = log(complex_abs2(fftwin[k])); + } + } else { + return -1; + } + } + + free(pf_WINDOW); + free(win_temp); + free(win_buf); + free(eph_buf); + free(win); + free(fftwin); + free(fft_buf); + + return 1; +} + +int Spectrum::get_spc(float* output) { + std::memcpy((void*)output, (void*)pf_SPC, \ + i_NumFrq * i_NumFrm * sizeof(float)); + free(pf_SPC); + return 1; +} + +int Spectrum::write_spc() { + FILE* fp; + fp = fopen("spectrum.txt", "w"); + int n, m; + for (n = 0; n < i_NumFrm; n++) { + for (m = 0; m < i_NumFrq; m++) { + fprintf(fp, "%4.6f ", pf_SPC[n * i_NumFrq + m]); + } + fprintf(fp, "\n"); + } + fclose(fp); + return 1; +} + +} // namespace delta diff --git a/athena/transform/feats/ops/kernels/spectrum.h b/athena/transform/feats/ops/kernels/spectrum.h new file mode 100644 index 00000000..517890ce --- /dev/null +++ b/athena/transform/feats/ops/kernels/spectrum.h @@ -0,0 +1,94 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef DELTA_LAYERS_OPS_KERNELS_SPECTRUM_H_ +#define DELTA_LAYERS_OPS_KERNELS_SPECTRUM_H_ + +#include +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/platform/logging.h" + +#include "kernels/complex_defines.h" +#include "kernels/support_functions.h" + +using namespace tensorflow; // NOLINT + +namespace delta { +class Spectrum { + private: + float window_length_sec_; + float frame_length_sec_; + + float f_SamRat; + int i_WinLen; + int i_FrmLen; + int i_NumFrm; + int i_NumFrq; + int i_FFTSiz; + float f_PreEph; + char s_WinTyp[40]; + int i_OutTyp; // 1: PSD, 2:log(PSD) + int i_snip_edges; + int i_raw_energy; + bool i_remove_dc_offset; + bool i_is_fbank; + float i_dither; + + float* pf_WINDOW; + float* pf_SPC; + + xcomplex* win; + float* win_buf; + float* eph_buf; + float* win_temp; + xcomplex* fftwin; + float* fft_buf; + + public: + Spectrum(); + + void set_window_length_sec(float window_length_sec); + + void set_frame_length_sec(float frame_length_sec); + + void set_output_type(int output_type); + + void set_snip_edges(int snip_edges); + + void set_raw_energy(int raw_energy); + + void set_preEph(float preEph); + + void set_window_type(char* window_type); + + void set_is_fbank(bool is_fbank); + + void set_remove_dc_offset(bool remove_dc_offset); + + void set_dither(float dither); + + int init_spc(int input_size, float sample_rate); + + int proc_spc(const float* mic_buf, int input_size); + + int get_spc(float* output); + + int write_spc(); + + TF_DISALLOW_COPY_AND_ASSIGN(Spectrum); +}; +} // namespace delta +#endif // DELTA_LAYERS_OPS_KERNELS_SPECTRUM_H_ diff --git a/athena/transform/feats/ops/kernels/spectrum_op.cc b/athena/transform/feats/ops/kernels/spectrum_op.cc new file mode 100644 index 00000000..d6afecea --- /dev/null +++ b/athena/transform/feats/ops/kernels/spectrum_op.cc @@ -0,0 +1,113 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "kernels/spectrum.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/status.h" +#include + +namespace delta { + +class SpecOp : public OpKernel { + public: + explicit SpecOp(OpKernelConstruction* context) : OpKernel(context) { + OP_REQUIRES_OK(context, context->GetAttr("window_length", &window_length_)); + OP_REQUIRES_OK(context, context->GetAttr("frame_length", &frame_length_)); + OP_REQUIRES_OK(context, context->GetAttr("output_type", &output_type_)); + OP_REQUIRES_OK(context, context->GetAttr("snip_edges", &snip_edges_)); + OP_REQUIRES_OK(context, context->GetAttr("raw_energy", &raw_energy_)); + OP_REQUIRES_OK(context, context->GetAttr("preEph_coeff", &preEph_coeff_)); + OP_REQUIRES_OK(context, context->GetAttr("window_type", &window_type_)); + OP_REQUIRES_OK(context, context->GetAttr("remove_dc_offset", &remove_dc_offset_)); + OP_REQUIRES_OK(context, context->GetAttr("is_fbank", &is_fbank_)); + OP_REQUIRES_OK(context, context->GetAttr("dither", &dither_)); + } + + void Compute(OpKernelContext* context) override { + const Tensor& input_tensor = context->input(0); + OP_REQUIRES(context, input_tensor.dims() == 1, + errors::InvalidArgument("input signal must be 1-dimensional", + input_tensor.shape().DebugString())); + + const Tensor& sample_rate_tensor = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(sample_rate_tensor.shape()), + errors::InvalidArgument( + "Input sample rate should be a scalar tensor, got ", + sample_rate_tensor.shape().DebugString(), " instead.")); + const float sample_rate = sample_rate_tensor.scalar()(); + + Spectrum cls_spc; + char* window_type = const_cast(window_type_.c_str()); + cls_spc.set_window_length_sec(window_length_); + cls_spc.set_frame_length_sec(frame_length_); + cls_spc.set_output_type(output_type_); + cls_spc.set_snip_edges(snip_edges_); + cls_spc.set_raw_energy(raw_energy_); + cls_spc.set_preEph(preEph_coeff_); + cls_spc.set_window_type(window_type); + cls_spc.set_remove_dc_offset(remove_dc_offset_); + cls_spc.set_is_fbank(is_fbank_); + cls_spc.set_dither(dither_); + + // shape + const int L = input_tensor.dim_size(0); + OP_REQUIRES(context, cls_spc.init_spc(L, sample_rate), + errors::InvalidArgument( + "spectrum_class initialization failed for length ", L, + " and sample rate ", sample_rate)); + + Tensor* output_tensor = nullptr; + int i_WinLen = static_cast(window_length_ * sample_rate); + int i_FrmLen = static_cast(frame_length_ * sample_rate); + int i_NumFrm = (L - i_WinLen) / i_FrmLen + 1; + int i_snip_edges = snip_edges_; + if (i_snip_edges == 2) + i_NumFrm = (L + i_FrmLen / 2) / i_FrmLen; + if (i_NumFrm < 1) + i_NumFrm = 1; + int i_FrqNum = static_cast(pow(2.0f, ceil(log2(i_WinLen))) / 2 + 1); + OP_REQUIRES_OK( + context, context->allocate_output(0, TensorShape({i_NumFrm, i_FrqNum}), + &output_tensor)); + + const float* input_flat = input_tensor.flat().data(); + float* output_flat = output_tensor->flat().data(); + + int ret; + ret = cls_spc.proc_spc(input_flat, L); + ret = cls_spc.get_spc(output_flat); + } + + private: + float window_length_; + float frame_length_; + int output_type_; + int snip_edges_; + int raw_energy_; + float preEph_coeff_; + string window_type_; + bool remove_dc_offset_; + bool is_fbank_; + float dither_; +}; + +REGISTER_KERNEL_BUILDER(Name("Spectrum").Device(DEVICE_CPU), SpecOp); + +} // namespace delta diff --git a/athena/transform/feats/ops/kernels/spectrum_op_test.py b/athena/transform/feats/ops/kernels/spectrum_op_test.py new file mode 100644 index 00000000..39a83f0a --- /dev/null +++ b/athena/transform/feats/ops/kernels/spectrum_op_test.py @@ -0,0 +1,68 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" spectrum Op unit-test """ +import os +from pathlib import Path + +import numpy as np +import tensorflow as tf +from absl import logging + +from athena.transform.feats.ops import py_x_ops + + +class SpecOpTest(tf.test.TestCase): + """ spectrum op unittest""" + + def setUp(self): + """set up""" + self.wavpath = str( + Path(os.environ["MAIN_ROOT"]).joinpath("delta/layers/ops/data/sm1_cln.wav") + ) + + def tearDown(self): + """tear down""" + + def test_spectrum(self): + """ test spectrum op""" + with self.session(use_gpu=False, force_gpu=False): + sample_rate, input_data = feat_lib.load_wav(self.wavpath, sr=16000) + logging.info( + f"input shape: {input_data.shape}, sample rate dtype: {sample_rate.dtype}" + ) + self.assertEqual(sample_rate, 16000) + + output = py_x_ops.spectrum(input_data, sample_rate) + + # pylint: disable=bad-whitespace + output_true = np.array( + [ + [-16.863441, -16.910473, -17.077059, -16.371634, -16.845686], + [-17.922068, -20.396345, -19.396944, -17.331493, -16.118851], + [-17.017776, -17.551350, -20.332376, -17.403994, -16.617926], + [-19.873854, -17.644503, -20.679525, -17.093716, -16.535091], + [-17.074402, -17.295971, -16.896650, -15.995432, -16.560730], + ] + ) + # pylint: enable=bad-whitespace + + self.assertEqual(tf.rank(output).eval(), 2) + logging.info("Shape of spectrum: {}".format(output.shape)) + self.assertAllClose(output.eval()[4:9, 4:9], output_true) + + +if __name__ == "__main__": + tf.test.main() diff --git a/athena/transform/feats/ops/kernels/speed_op.cc b/athena/transform/feats/ops/kernels/speed_op.cc new file mode 100644 index 00000000..5fee5936 --- /dev/null +++ b/athena/transform/feats/ops/kernels/speed_op.cc @@ -0,0 +1,84 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "kernels/resample.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/status.h" + +namespace delta { + +class SpeedOp : public OpKernel { + public: + explicit SpeedOp(OpKernelConstruction* context) : OpKernel(context) { + OP_REQUIRES_OK(context, + context->GetAttr("lowpass_filter_width", &lowpass_filter_width_)); + } + + void Compute(OpKernelContext* context) override { + const Tensor& input_tensor = context->input(0); + OP_REQUIRES(context, input_tensor.dims() == 1, + errors::InvalidArgument("input signal must be 1-dimensional", + input_tensor.shape().DebugString())); + const Tensor& sample_rate_tensor = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(sample_rate_tensor.shape()), + errors::InvalidArgument( + "Input sample_rate should be a scalar tensor, got ", + sample_rate_tensor.shape().DebugString(), " instead.")); + const Tensor& resample_rate_tensor = context->input(2); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(resample_rate_tensor.shape()), + errors::InvalidArgument( + "Resample sample_rate should be a scalar tensor, got ", + resample_rate_tensor.shape().DebugString(), " instead.")); + const int sample_rate = static_cast(sample_rate_tensor.scalar()()); + const int resample_freq = static_cast(resample_rate_tensor.scalar()()); + const float* input_flat = input_tensor.flat().data(); + const int L = input_tensor.dim_size(0); + + lowpass_cutoff_ = min(resample_freq / 2, sample_rate / 2); + LinearResample cls_resample_(sample_rate, resample_freq, + lowpass_cutoff_, + lowpass_filter_width_); + vector waveform(L); + for (int i = 0; i < L; i++){ + waveform[i] = static_cast(input_flat[i]); + } + vector downsampled_wave; + cls_resample_.Resample(waveform, false, &downsampled_wave); + int output_length = downsampled_wave.size(); + Tensor* output_tensor = nullptr; + OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape({1, output_length}), + &output_tensor)); + float* output_flat = output_tensor->flat().data(); + for (int j = 0; j < output_length; j++) + output_flat[j] = downsampled_wave[j]; + + std::vector().swap(downsampled_wave); + std::vector().swap(waveform); + cls_resample_.Reset(); + } + + private: + float lowpass_cutoff_; + int lowpass_filter_width_; +}; + +REGISTER_KERNEL_BUILDER(Name("Speed").Device(DEVICE_CPU), SpeedOp); + +} // namespace delta \ No newline at end of file diff --git a/athena/transform/feats/ops/kernels/support_functions.cc b/athena/transform/feats/ops/kernels/support_functions.cc new file mode 100644 index 00000000..224be139 --- /dev/null +++ b/athena/transform/feats/ops/kernels/support_functions.cc @@ -0,0 +1,725 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "kernels/support_functions.h" + +#include +#include +#include +#include + +#define PI (3.141592653589793) + +#define SWAP(a, b) \ + tempr = (a); \ + (a) = (b); \ + (b) = tempr + +namespace delta { +/* compute mean */ +float compute_mean(float* input, int i_size) { + int n; + float f_mean = 0.0; + for (n = 0; n < i_size; n++) { + f_mean = f_mean + input[n] / i_size; + } + return f_mean; +} + +/* compute variance */ +float compute_var(float* input, int i_size) { + int n; + float f_var = 0.0; + float f_mean = compute_mean(input, i_size); + + for (n = 0; n < i_size; n++) { + f_var = f_var + pow(input[n] - f_mean, 2.0) / i_size; + } + return f_var; +} + +/* compute autocorrelation */ +int compute_autoc(float* autoc, float* input, int i_size, int L) { + if (i_size < L) { + printf( + "L should be less than i_size in autocorrelation computation, set " + "L=i_size!\n"); + L = i_size; + } + int n, m; + float f_mean = compute_mean(input, i_size); + float f_var = compute_var(input, i_size); + float tmp; + + for (m = 0; m <= L; m++) { + autoc[m] = 0.0; + for (n = 0; n < i_size - m; n++) { + autoc[m] = autoc[m] + (input[n] - f_mean) * (input[n + m] - f_mean); + } + // autoc[m] = (1.0/(i_size-m)) * autoc[m]; + // autoc[m] = autoc[m] / f_var; + if (m == 0) { + tmp = autoc[0]; + } + autoc[m] = autoc[m] / tmp; + } + return 0; +} + +/* compute sign */ +int compute_sign(float x) { return (x > 0) - (x < 0); } + +/* generate window */ +int gen_window(float* w, int L, char* typ) { + float* pn = static_cast(malloc(sizeof(float) * L)); + int n; + for (n = 0; n < L; n++) { + pn[n] = 2 * PI * n / (L - 1); + } + + if (strcmp(typ, "rect") == 0) { + for (n = 0; n < L; n++) { + w[n] = 1.0; + } + } else if (strcmp(typ, "tria") == 0) { + for (n = 0; n < (L - 1) / 2; n++) { + w[n] = n / ((L - 1) / 2); + } + for (n = (L - 1) / 2 + 1; n < L; n++) { + w[n] = w[L - n - 1]; + } + } else if (strcmp(typ, "hann") == 0) { + for (n = 0; n < L; n++) { + w[n] = 0.5 * (1 - cos(pn[n])); + } + } else if (strcmp(typ, "hamm") == 0) { + for (n = 0; n < L; n++) { + w[n] = 0.54 - 0.46 * cos(pn[n]); + } + } else if (strcmp(typ, "povey") == 0) { + for (n = 0; n < L; n++){ + w[n] = pow(0.5 - 0.5 * cos(pn[n]), 0.85); + } + } else if (strcmp(typ, "blac") == 0) { + for (n = 0; n < L; n++) { + w[n] = 0.42 - 0.5 * cos(pn[n]) + 0.08 * cos(2 * pn[n]); + } + } else { + printf("Window type not support!\n"); + return -1; + } + free(pn); + return 0; +} + +/* do pre-emphasis */ +int do_preemphasis(float coef, float* buf, const float* input, int L) { + /* a = filter([1 coef], 1, b) */ + if (coef == 0.0) { + memcpy(buf, input, sizeof(float) * L); + return 0; + } + + int n; + for (n = 0; n < L + 1; n++) { + if (n == 0) { + buf[n] = input[n] * 1.0; + } else if (n == L) { + buf[n] = input[n - 1] * (-1) * coef; + } else { + buf[n] = input[n] * 1.0 + input[n - 1] * (-1) * coef; + } + } + + return 0; +} + +/* Hertz to Bark conversion */ +float hz2bark(float f) { + double z; + z = 6.0 * asinh(static_cast(f) / 600.0); + return static_cast(z); +} + +/* Bark to Hertz conversion */ +float bark2hz(float z) { + double f; + f = 600.0 * sinh(static_cast(z) / 6.0); + return static_cast(f); +} + +/* Hertz to Mel conversion */ +float hz2mel(float f) { + double z; + z = 2595.0 * log10(1 + static_cast(f) / 700.0); + return static_cast(z); +} + +/* Mel to Hertz conversion */ +float mel2hz(float z) { + double f; + f = 700.0 * (pow(10, static_cast(z) / 2595.0) - 1); + return static_cast(f); +} + +/* generate frequency scale warping */ +int gen_warpweights(float fs, int ncep, int nfrq, char* ftyp, int nfft, + float* wts) { + float bwidth = 1.0; + float minfrq = 100.0; /* lower bound in Hz */ + float maxfrq = fs / 2; /* upper bound in Hz */ + + if (strcmp(ftyp, "bark") == 0) { + float minbark = hz2bark(minfrq); + float nyqbark = hz2bark(maxfrq) - minbark; + float stpbark = nyqbark / (ncep - 1); + int n, f; + float f_bark_mid, lof, hif, tmp; + float* binbark = + static_cast(malloc(sizeof(float) * (nfft / 2 + 1))); + + for (n = 0; n < nfft / 2 + 1; n++) { + binbark[n] = hz2bark(n * fs / nfft); + } + for (f = 0; f < ncep; f++) { + f_bark_mid = minbark + f * stpbark; + for (n = 0; n < nfft / 2 + 1; n++) { + lof = (binbark[n] - f_bark_mid) / bwidth - 0.5; + hif = (binbark[n] - f_bark_mid) / bwidth + 0.5; + tmp = fmin(hif, -2.5 * lof); + tmp = fmin(tmp, 0); + wts[f * (nfft / 2 + 1) + n] = pow(10, tmp); + } + } + + free(binbark); + return 0; + } else if (strcmp(ftyp, "mel") == 0) { + float minmel = hz2mel(minfrq); + float maxmel = hz2mel(maxfrq); + int n, f; + float tmp, fss1, fss2, fss3, loslope, hislope; + float* fftfrq = static_cast(malloc(sizeof(float) * (nfft / 2 + 1))); + float* binfrq = static_cast(malloc(sizeof(float) * (ncep + 2))); + + for (n = 0; n < nfft / 2 + 1; n++) { + fftfrq[n] = n * fs / nfft; + } + for (f = 0; f < ncep + 2; f++) { + tmp = minmel + static_cast(f) / (ncep + 1) * (maxmel - minmel); + binfrq[f] = mel2hz(tmp); + } + + for (f = 0; f < ncep; f++) { + fss1 = binfrq[f]; + fss2 = binfrq[f + 1]; + fss3 = binfrq[f + 2]; + fss1 = fss2 + bwidth * (fss1 - fss2); + fss3 = fss2 + bwidth * (fss3 - fss2); + for (n = 0; n < nfft / 2 + 1; n++) { + loslope = (fftfrq[n] - fss1) / (fss2 - fss1); + hislope = (fss3 - fftfrq[n]) / (fss3 - fss2); + tmp = fmin(loslope, hislope); + wts[f * (nfft / 2 + 1) + n] = fmax(0.0f, tmp); + } + } + + free(fftfrq); + free(binfrq); + return 0; + } +} + +/* generate DCT matrix */ +int gen_dctmatrix(int ncep, float* dctm) { + int n, m; + float tmp1, tmp2; + tmp1 = static_cast(PI / 2 / ncep); + tmp2 = sqrt(2.0f / ncep); + for (n = 0; n < ncep; n++) { + for (m = 0; m < ncep; m++) { + dctm[n * ncep + m] = + cos(static_cast(n) * (2 * m + 1) * tmp1) * tmp2; + } + } + return 0; +} + +/* lift cepstrum */ +int lift_cepstrum(float lft_exp, int ncep, int nfrm, float* x, float* y) { + int n, m; + float liftwts; + if (lft_exp <= 0) { + memcpy(y, x, sizeof(float) * ncep * nfrm); + } else { + for (n = 0; n < ncep; n++) { + if (n == 0) { + liftwts = 1.0; + } else { + liftwts = pow(n, lft_exp); + } + for (m = 0; m < nfrm; m++) { + y[m * ncep + n] = liftwts * x[m * ncep + n]; + } + } + } + return 0; +} + +/* cepstrum mean normalization */ +int do_ceps_mean_norm(int ncep, int nfrm, float* x, float* y) { + int n, m; + float cmn; + + for (n = 0; n < ncep; n++) { + if (n == 0) { + for (m = 0; m < nfrm; m++) { + y[m * ncep + n] = x[m * ncep + n]; + } + } else { + cmn = 0.0; + for (m = 0; m < nfrm; m++) { + cmn = cmn + x[m * ncep + n] / nfrm; + } + + for (m = 0; m < nfrm; m++) { + y[m * ncep + n] = x[m * ncep + n] - cmn; + } + } + } + + return 0; +} + +/* compute delta coefficients */ +int compute_delta(int nw, int ncep, int nfrm, float* x, float* y) { + if (nw % 2 == 0) /* make nw odd number */ { + nw = nw + 1; + } + + int hlen = (nw - 1) / 2; + float* win = static_cast(malloc(sizeof(float) * nw)); + int n, m, k; + float f_tmp, f_tmp2; + int i_tmp, i_dist; + + for (n = 0; n < nw; n++) { + win[n] = n - hlen; /* left-right flipped win */ + } + + for (n = 0; n < ncep; n++) { + for (m = 0; m < nfrm; m++) { + if (m == 0) { + y[n * nfrm + m] = x[n * nfrm + m]; + } else if (m == nfrm - 1) { + y[n * nfrm + m] = x[n * nfrm + m]; + } else { + if (m < hlen) { + i_dist = m; + } else if (m > nfrm - 1 - hlen) { + i_dist = nfrm - 1 - m; + } else { + i_dist = hlen; + } + f_tmp = 0.0; + f_tmp2 = 0.0; + for (k = 0; k <= 2 * i_dist; k++) { + f_tmp = f_tmp + win[hlen - i_dist + k] * x[m - i_dist + k]; + f_tmp2 = f_tmp2 + (i_dist - k) * (i_dist - k); + } + y[n * nfrm + m] = f_tmp / f_tmp2; + } + } + } + + free(win); + return 0; +} + +/* naive DFT */ +int naive_dft(xcomplex* input, xcomplex* output, int inverse, int N) { + float coef = (inverse == 1 ? 2 : -2) * PI; + int n, m; + float angle; + for (int n = 0; n < N; n++) { + output[n].r = 0.0; + output[n].i = 0.0; + for (m = 0; m < N; m++) { + angle = coef * static_cast(m * n % N) / N; + output[n].r += input[m].r * cos(angle) - input[m].i * sin(angle); + output[n].i += input[m].r * sin(angle) + input[m].i * cos(angle); + } + + if (inverse == 1) { + output[n] = complex_real_complex_mul(1.0f / N, output[n]); + } + } + return 0; +} + +/* do loudness equalization and cube root compression */ +int do_EqlLoudness(float fs, char* ftyp, int ncep, int nfrm, float* x, + float* y) { + int n, m; + float tmp, tmp2, zmax; + float* bandcfhz = static_cast(malloc(sizeof(float) * ncep)); + float* eql = static_cast(malloc(sizeof(float) * ncep)); + if (strcmp(ftyp, "bark") == 0) { + zmax = hz2bark(fs); + for (n = 0; n < ncep; n++) { + tmp = static_cast(n) * zmax / (ncep - 1); + bandcfhz[n] = bark2hz(tmp); + } + } else if (strcmp(ftyp, "mel") == 0) { + zmax = hz2mel(fs); + for (n = 0; n < ncep; n++) { + tmp = static_cast(n) * zmax / (ncep - 1); + bandcfhz[n] = mel2hz(tmp); + } + } else { + printf("ftype not support in do_Eqlloudness\n"); + return -1; + } + + /* Hynek's magic equal-loudness-curve formula */ + for (n = 0; n < ncep; n++) { + tmp = pow(bandcfhz[n], 2); + tmp2 = tmp + 1.6e5; + eql[n] = pow(tmp / tmp2, 2) * (tmp + 1.44e6) / (tmp + 9.61e6); + } + + /* weight the critical band */ + for (n = 0; n < nfrm; n++) { + for (m = 0; m < ncep; m++) { + tmp = eql[m] * x[n * ncep + m]; + y[n * ncep + m] = pow(tmp, 1.0f / 3.0f); + } + } + + /* replicate 1st and last band due to unreliability */ + for (n = 0; n < nfrm; n++) { + y[n * ncep + 0] = y[n * ncep + 1]; + y[n * ncep + ncep - 1] = y[n * ncep + ncep - 2]; + } + + free(bandcfhz); + free(eql); + return 0; +} + +/* convert LPC coeff into cepstrum */ +int do_lpc2cep(int ncep, int nfrm, float* x, float* y) { + int n, m, k; + float tmp, sum; + float* nx = static_cast(malloc(sizeof(float) * ncep * nfrm)); + + /* 1st cep band is log(err) from Durbin */ + for (n = 0; n < nfrm; n++) { + y[n * ncep + 0] = -log(x[n * ncep + 0]); + } + + /* renormalize lpc coeff */ + for (n = 0; n < nfrm; n++) { + nx[n * ncep + 0] = 1.0f; + for (m = 1; m < ncep; m++) { + nx[n * ncep + m] = x[n * ncep + m] / x[n * ncep + 0]; + } + } + + for (n = 0; n < nfrm; n++) { + for (m = 1; m < ncep; m++) { + sum = 0.0f; + for (k = 1; k <= m; k++) { + sum = sum + (m - k) * nx[n * ncep + k] * y[n * ncep + m - k]; + } + y[n * ncep + m] = -nx[n * ncep + m] - sum / m; + } + } + + free(nx); + return 0; +} + +/* Levinson durbin recursion */ +int do_levinson(int pord, float* r, float* a) { + float* as = static_cast(malloc(sizeof(float) * pord * pord)); + float* P = static_cast(malloc(sizeof(float) * pord)); + int n, m; + float sum; + + /* for m = 1 */ + as[0] = -r[1] / r[0]; + P[0] = r[0] * (1 - pow(as[0], 2)); + /* for m = 2:pord */ + for (n = 1; n < pord; n++) { + sum = 0.0f; + for (m = 0; m < n; m++) { + sum = sum + as[(n - 1) * pord + m] * r[n - m]; + } + as[n * pord + n] = -(r[n + 1] + sum) / P[n - 1]; + for (m = 0; m < n; m++) { + as[n * pord + m] = as[(n - 1) * pord + m] + + as[n * pord + n] * as[(n - 1) * pord + n - 1 - m]; + } + P[n] = P[n - 1] * (1 - pow(as[n * pord + n], 2)); + } + a[0] = 1.0f; + for (n = 0; n < pord; n++) { + a[n + 1] = as[(pord - 1) * pord + n]; + } + + free(as); + free(P); + return 0; +} + +/* compute lpc coeff */ +int compute_lpc(int ncep, int nfrm, int pord, float* x, float* y) { + int n, m, nband = (ncep - 1) * 2; + xcomplex* nx = static_cast(malloc(sizeof(xcomplex) * nband)); + xcomplex* fx = static_cast(malloc(sizeof(xcomplex) * nband)); + float* xr = static_cast(malloc(sizeof(float) * ncep)); + float* yr = static_cast(malloc(sizeof(float) * (pord + 1))); + + for (n = 0; n < nfrm; n++) { + for (m = 0; m < ncep; m++) { + nx[m].r = x[n * ncep + m]; + nx[m].i = 0.0f; + } + for (m = ncep; m < nband; m++) { + nx[m].r = x[n * ncep + nband - m]; + nx[m].i = 0.0f; + } + + naive_dft(nx, fx, 1, nband); + + for (m = 0; m < ncep; m++) { + xr[m] = fx[m].r; + } + + do_levinson(pord, xr, yr); + + for (m = 0; m < pord + 1; m++) { + y[n * (pord + 1) + m] = yr[m]; + } + } + + free(nx); + free(fx); + free(xr); + free(yr); + return 0; +} + +/* Radix-2 DIT FFT */ +/* isign=-1 ==> FFT, isign=1 ==> IFFT */ +int dit_r2_fft(xcomplex* input, xcomplex* output, float* in_buf, int N, int isign) { + float wtemp, wr, wpr, wpi, wi, theta; + float tempr, tempi; + int i = 0, j = 0, n = 0, k = 0, m = 0, istep, mmax; + float* out_buf; + float den; + if (isign == -1) + den = 1.0f; + else + den = static_cast(N); + + for (i = 0; i < N; i++) { + in_buf[i * 2] = input[i].r; + in_buf[i * 2 + 1] = input[i].i; + } + + out_buf = &in_buf[0] - 1; + n = N * 2; + j = 1; + + // do the bit-reversal + for (i = 1; i < n; i += 2) { + if (j > i) { + SWAP(out_buf[j], out_buf[i]); + SWAP(out_buf[j + 1], out_buf[i + 1]); + } + m = n >> 1; + while (m >= 2 && j > m) { + j -= m; + m >>= 1; + } + j += m; + } + + // calculate the FFT + mmax = 2; + while (n > mmax) { + istep = mmax << 1; + theta = isign * (2 * PI / mmax); + wtemp = sin(0.5 * theta); + wpr = -2.0 * wtemp * wtemp; + wpi = sin(theta); + wr = 1.0; + wi = 0.0; + for (m = 1; m < mmax; m += 2) { + for (i = m; i <= n; i += istep) { + j = i + mmax; + tempr = wr * out_buf[j] - wi * out_buf[j + 1]; + tempi = wr * out_buf[j + 1] + wi * out_buf[j]; + out_buf[j] = out_buf[i] - tempr; + out_buf[j + 1] = out_buf[i + 1] - tempi; + out_buf[i] = out_buf[i] + tempr; + out_buf[i + 1] = out_buf[i + 1] + tempi; + } + wtemp = wr; + wr += wtemp * wpr - wi * wpi; + wi += wtemp * wpi + wi * wpr; + } + mmax = istep; + } + + for (i = 0; i < N; i++) { + output[i].r = out_buf[i * 2 + 1] / den; + output[i].i = out_buf[i * 2 + 2] / den; + } + return 0; +} + +/* compute energy of frame */ +float compute_energy(const float* input, int L){ + float energy = 0; + for (int i = 0; i < L; i++){ + energy += input[i] * input[i]; + } + return energy; +} + +/* do pre_emphasis on frame */ +int do_frame_preemphasis(float* input, float* output, int i_size, float coef){ + if (coef == 0.0){ + memcpy(output, input, sizeof(float) * i_size); + return 0; + } + memcpy(output, input, sizeof(float) * i_size); + for (int i = i_size - 1; i > 0; i--) + output[i] -= coef * output[i-1]; + output[0] -= coef * output[0]; + return 0; +} + +/* return subvector */ +std::vector sub_vector(const std::vector &input, int begin, int length){ + std::vector res(length); + for (int i = 0; i < length; i++){ + res[i] = input[i + begin]; + } + return res; +} + +std::vector> sub_matrix(const std::vector> &input, + int row_begin, int n_rows, int col_begin, int n_cols){ + std::vector> res; + res.resize(n_rows); + for (int i = 0; i < n_rows; i++){ + res[i].resize(n_cols); + for (int j = 0; j < n_cols; j++) + res[i][j] = input[row_begin + i][col_begin + j]; + } + return res; +} + +std::vector add_mat_vec(BaseFloat alpha, const std::vector> &M, + const std::vector &v, BaseFloat beta){ + int row, col, length; + row = M.size(); + col = M[0].size(); + length = v.size(); + std::vector res(row, 0); + for (int i = 0; i < row; i++) + { + float tmp = 0.0; + for (int j = 0; j < col; j++){ + tmp += M[i][j] * v[j]; + } + res[i] = alpha * tmp + beta * res[i]; + } + return res; +} + +/*return gcd */ +int Gcd(int m, int n) { + if (m == 0 || n == 0) { + if (m == 0 && n == 0) { // gcd not defined, as all integers are divisors. + std::cerr << "Undefined GCD since m = 0, n = 0."; + } + return (m == 0 ? (n > 0 ? n : -n) : ( m > 0 ? m : -m)); + // return absolute value of whichever is nonzero + } + while (1) { + m %= n; + if (m == 0) return (n > 0 ? n : -n); + n %= m; + if (n == 0) return (m > 0 ? m : -m); + } +} + +int Lcm(int m, int n) { + assert(m > 0 && n > 0); + int gcd = Gcd(m, n); + return gcd * (m/gcd) * (n/gcd); +} + +BaseFloat VecVec(const vector &a, const vector &b){ + assert(a.size() == b.size()); + BaseFloat res = 0; + for (int i = 0; i < a.size(); ++i) + res += a[i] * b[i]; + return res; +} + +/* do dither */ +void do_dither(float* input, int i_size, float dither_value){ + if (dither_value == 0.0) + return; + RandomState rstate; + for (int i = 0; i < i_size; i++) + input[i] += RandGauss(&rstate) * dither_value; +} + +int Rand(struct RandomState* state) { +#if !defined(_POSIX_THREAD_SAFE_FUNCTIONS) + // On Windows and Cygwin, just call Rand() + return rand(); +#else + if (state) { + return rand_r(&(state->seed)); + } else { + std::lock_guard lock(_RandMutex); + return rand(); + } +#endif +} + +RandomState::RandomState() { + // we initialize it as Rand() + 27437 instead of just Rand(), because on some + // systems, e.g. at the very least Mac OSX Yosemite and later, it seems to be + // the case that rand_r when initialized with rand() will give you the exact + // same sequence of numbers that rand() will give if you keep calling rand() + // after that initial call. This can cause problems with repeated sequences. + // For example if you initialize two RandomState structs one after the other + // without calling rand() in between, they would give you the same sequence + // offset by one (if we didn't have the "+ 27437" in the code). 27437 is just + // a randomly chosen prime number. + seed = Rand() + 27437; +} + +} // namespace delta diff --git a/athena/transform/feats/ops/kernels/support_functions.h b/athena/transform/feats/ops/kernels/support_functions.h new file mode 100644 index 00000000..86996243 --- /dev/null +++ b/athena/transform/feats/ops/kernels/support_functions.h @@ -0,0 +1,172 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef DELTA_LAYERS_OPS_KERNELS_SUPPORT_FUNCTIONS_H_ +#define DELTA_LAYERS_OPS_KERNELS_SUPPORT_FUNCTIONS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernels/complex_defines.h" + +#ifndef M_PI +#define M_PI 3.1415926535897932384626433832795 +#endif + +#ifndef M_2PI +#define M_2PI 6.283185307179586476925286766559005 +#endif + +using namespace std; + +namespace delta { +typedef float BaseFloat; +/* compute mean */ +float compute_mean(float* input, int i_size); + +/* compute variance */ +float compute_var(float* input, int i_size); + +/* compute autocorrelation */ +int compute_autoc(float* autoc, float* input, int i_size, int L); + +/* compute sign */ +int compute_sign(float x); + +/* generate window */ +int gen_window(float* w, int L, char* typ); + +/* do pre-emphasis */ +int do_preemphasis(float coef, float* buf, const float* input, int i_size); + +/* Hertz to Bark conversion */ +float hz2bark(float f); + +/* Bark to Hertz conversion */ +float bark2hz(float z); + +/* Hertz to Mel conversion */ +float hz2mel(float f); + +/* Mel to Hertz conversion */ +float mel2hz(float z); + +/* generate frequency scale warping */ +int gen_warpweights(float fs, int ncep, int nfrq, char* ftyp, int nfft, + float* wts); + +/* generate DCT matrix */ +int gen_dctmatrix(int ncep, float* dctm); + +/* lift cepstrum */ +int lift_cepstrum(float lft_exp, int ncep, int nfrm, float* x, float* y); + +/* cepstrum mean normalization */ +int do_ceps_mean_norm(int ncep, int nfrm, float* x, float* y); + +/* compute delta coefficients */ +int compute_delta(int nw, int ncep, int nfrm, float* x, float* y); + +/* naive DFT */ +int naive_dft(xcomplex* input, xcomplex* output, int inverse, int N); + +/* do loudness equalization and cube root compression */ +int do_EqlLoudness(float fs, char* ftyp, int ncep, int nfrm, float* x, + float* y); + +/* convert lpc coeff into cepstrum */ +int do_lpc2cep(int ncep, int nfrm, float* x, float* y); + +/* Levinson durbin recursion */ +int do_levinson(int pord, float* r, float* a); + +/* compute lpc coeff */ +int compute_lpc(int ncep, int nfrm, int pord, float* x, float* y); + +/* radix-2 DIT FFT */ +int dit_r2_fft(xcomplex* input, xcomplex* output, float* in_buf, int N, int isign); + +/* compute energy of frame */ +float compute_energy(const float* input, int L); + +/* do frame_pre_emphasis */ +int do_frame_preemphasis(float* input, float* output, int i_size, float coef); + +/* add dither */ +void do_dither(float* input, int i_size, float dither_value); + +/* return subvector */ +std::vector sub_vector(const std::vector &input, int begin, int length); + +std::vector> sub_matrix(const std::vector> &input, + int row_begin, int n_rows, int col_begin, int n_cols); + +std::vector add_mat_vec(BaseFloat alpha, const std::vector> &M, + const std::vector &v, BaseFloat beta); +/* return gcd */ +int Gcd(int m, int n); + +/*return lcm */ +int Lcm(int m, int n); + +BaseFloat VecVec(const vector &a, const vector &b); + +static inline bool ApproxEqual(BaseFloat a, BaseFloat b, + BaseFloat relative_tolerance = 0.001) { + // a==b handles infinities. + if (a == b) return true; + BaseFloat diff = std::fabs(a-b); + if (diff == std::numeric_limits::infinity() + || diff != diff) return false; // diff is +inf or nan. + return (diff <= (relative_tolerance * (std::fabs(a)+std::fabs(b)))); +} + +// Returns a random integer between 0 and RAND_MAX, inclusive +int Rand(struct RandomState* state = NULL); + +// State for thread-safe random number generator +struct RandomState { + RandomState(); + unsigned seed; +}; + +inline double Log(double x) { return log(x); } + +/// Returns a random number strictly between 0 and 1. +inline float RandUniform(struct RandomState* state = NULL) { + return static_cast((Rand(state) + 1.0) / (RAND_MAX+2.0)); +} + +inline float RandGauss(struct RandomState* state = NULL) { + return static_cast(sqrtf (-2 * Log(RandUniform(state))) + * cosf(2*M_PI*RandUniform(state))); +} + +template +std::unique_ptr make_unique(Args&&... args) { + return std::unique_ptr(new T(std::forward(args)...)); +} + +} // namespace delta +#endif // DELTA_LAYERS_OPS_KERNELS_SUPPORT_FUNCTIONS_H_ + diff --git a/athena/transform/feats/ops/kernels/x_ops.cc b/athena/transform/feats/ops/kernels/x_ops.cc new file mode 100644 index 00000000..5c98371b --- /dev/null +++ b/athena/transform/feats/ops/kernels/x_ops.cc @@ -0,0 +1,292 @@ +/* Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/shape_inference.h" + +// See tensorflow/core/ops/audio_ops.cc +namespace delta { +namespace { + +using namespace tensorflow; // NOLINT +using tensorflow::shape_inference::DimensionHandle; +using tensorflow::shape_inference::InferenceContext; +using tensorflow::shape_inference::ShapeHandle; +using tensorflow::shape_inference::UnchangedShape; + + +//SetShapeFn has issue [https://github.com/tensorflow/tensorflow/issues/29643] + +Status SpectrumShapeFn(InferenceContext* c) { + ShapeHandle input_data; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &input_data)); + + int wav_len = c->Value(c->Dim(input_data, 0)); + int SampleRate = 16000; + float win_len, frm_len; + TF_RETURN_IF_ERROR(c->GetAttr("window_length", &win_len)); + TF_RETURN_IF_ERROR(c->GetAttr("frame_length", &frm_len)); + int i_WinLen = static_cast(win_len * SampleRate); + int i_FrmLen = static_cast(frm_len * SampleRate); + int i_NumFrm = (wav_len - i_WinLen) / i_FrmLen + 1; + int i_FrqNum = static_cast(pow(2.0f, ceil(log2(i_WinLen))) / 2 + 1); + c->set_output(0, c->Matrix(i_NumFrm, i_FrqNum)); + + return Status::OK(); +} + +Status PitchShapeFn(InferenceContext* c) { + ShapeHandle input_data; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &input_data)); + + int wav_len = c->Value(c->Dim(input_data, 0)); + int SampleRate = 16000; + float win_len, frm_len; + TF_RETURN_IF_ERROR(c->GetAttr("window_length", &win_len)); + TF_RETURN_IF_ERROR(c->GetAttr("frame_length", &frm_len)); + int i_WinLen = static_cast(win_len * SampleRate); + int i_FrmLen = static_cast(frm_len * SampleRate); + int i_NumFrm = (wav_len - i_WinLen) / i_FrmLen + 1; + int i_FrqNum = static_cast(pow(2.0f, ceil(log2(i_WinLen))) / 2 + 1); + c->set_output(0, c->Matrix(i_NumFrm, 2)); + + return Status::OK(); +} + +Status FbankShapeFn(InferenceContext* c) { + ShapeHandle spectrogram; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 3, &spectrogram)); + ShapeHandle unused; + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + + int32 filterbank_channel_count; + TF_RETURN_IF_ERROR( + c->GetAttr("filterbank_channel_count", &filterbank_channel_count)); + + DimensionHandle audio_channels = c->Dim(spectrogram, 0); + DimensionHandle spectrogram_length = c->Dim(spectrogram, 1); + + DimensionHandle output_channels = c->MakeDim(filterbank_channel_count); + + c->set_output( + 0, c->MakeShape({audio_channels, spectrogram_length, output_channels})); + return Status::OK(); +} + + + +} // namespace + +// TF: +// wav to spectrogram: +// https://github.com/tensorflow/tensorflow/examples/wav_to_spectrogram/wav_to_spectrogram.h +// audio ops: https://github.com/tensorflow/tensorflow/core/ops/audio_ops.cc +// DecodeWav: +// tensorflow/tensorflow/core/api_def/base_api/api_def_DecodeWav.pbtxt +// EncodeWav: +// tensorflow/tensorflow/core/api_def/base_api/api_def_EncodeWav.pbtxt +// AudioSpectrogram: +// https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/spectrogram_op.cc +// tensorflow/tensorflow/core/api_def/base_api/api_def_AudioSpectrogram.pbtxt +// Mfcc: +// https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/mfcc_op.cc +// tensorflow/tensorflow/core/api_def/base_api/api_def_Mfcc.pbtxt + +// TFLite +// mfcc: +// https://github.com/tensorflow/tensorflow/tensorflow/lite/kernels/internal/mfcc_mel_filterbank.h +// spectorgram: tensorflow/tensorflow/lite/kernels/internal/spectrogram.h + +REGISTER_OP("Spectrum") + .Input("input_data: float") + .Input("sample_rate: float") + .Attr("window_length: float = 0.025") + .Attr("frame_length: float = 0.010") + .Attr("window_type: string") + .Attr("output_type: int = 2") + .Attr("snip_edges: int = 1") + .Attr("raw_energy: int = 1") + .Attr("preEph_coeff: float = 0.97") + .Attr("remove_dc_offset: bool = true") + .Attr("is_fbank: bool = true") + .Attr("dither: float = 1.0") + .Output("output: float") + .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c){ + return Status::OK(); + }) +// .SetShapeFn(SpectrumShapeFn) + .Doc(R"doc( + Create spectrum feature files. + input_data: float, input wave, a tensor of shape [1, data_length]. + sample_rate: float, NB 8000, WB 16000 etc. + window_length: float, window length in second. + frame_length: float, frame length in second. + output_type: int, 1: PSD, 2: log(PSD). + raw_energy: int, 1: raw energy, 2: wined_energy. + output: float, PSD/logPSD features, [num_Frame, num_Subband]. + )doc"); + +REGISTER_OP("FramePow") + .Input("input_data: float") + .Input("sample_rate: float") + .Attr("snip_edges: int = 1") + .Attr("remove_dc_offset: bool = true") + .Attr("window_length: float = 0.025") + .Attr("frame_length: float = 0.010") + .Output("output: float") + .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c){ + return Status::OK(); + }) + .Doc(R"doc( + Create frame energy feature files. + input_data: float, input wave, a tensor of shape [1, data_length]. + sample_rate: float, NB 8000, WB 16000 etc. + window_length: float, window length in second. + frame_length: float, frame length in second. + output: float, frame energy features, [num_Frame]. + )doc"); + +REGISTER_OP("Pitch") + .Input("input_data: float") + .Input("sample_rate: int32") + .Attr("window_length: float = 0.025") + .Attr("frame_length: float = 0.010") + .Attr("snip_edges: bool = true") + .Attr("preemph_coeff: float = 0.0") + .Attr("min_f0: float = 50") + .Attr("max_f0: float = 400") + .Attr("soft_min_f0: float = 10.0") + .Attr("penalty_factor: float = 0.1") + .Attr("lowpass_cutoff: float = 1000") + .Attr("resample_freq: float = 4000") + .Attr("delta_pitch: float = 0.005") + .Attr("nccf_ballast: float = 7000") + .Attr("lowpass_filter_width: int = 1") + .Attr("upsample_filter_width: int = 5") + .Attr("max_frames_latency: int = 0") + .Attr("frames_per_chunk: int = 0") + .Attr("simulate_first_pass_online: bool = false") + .Attr("recompute_frame: int = 500") + .Attr("nccf_ballast_online: bool = false") + .Output("output: float") + .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c){ + return Status::OK(); + }) + .Doc(R"doc( + Create pitch feature files. + input_data: float, input wave, a tensor of shape [1, data_length]. + sample_rate: float, NB 8000, WB 16000 etc. + window_length: float, window length in second. + frame_length: float, frame length in second. + output: float, pitch features, [num_Frame, 2]. + )doc"); + +REGISTER_OP("Speed") + .Input("input_data: float") + .Input("sample_rate: int32") + .Input("resample_freq: int32") + .Attr("lowpass_filter_width: int = 1") + .Output("output: float") + .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c){ + return Status::OK(); + }) + .Doc(R"doc( + Create pitch feature files. + input_data: float, input wave, a tensor of shape [1, data_length]. + sample_rate: float, NB 8000, WB 16000 etc. + )doc"); + +REGISTER_OP("MfccDct") + .Input("fbank: float") + .Input("framepow: float") + .Input("sample_rate: int32") + .Attr("coefficient_count: int = 13") + .Attr("cepstral_lifter: float = 22") + .Attr("use_energy: bool = true") + .Output("output: float") + .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c){ + return Status::OK(); + }) + .Doc(R"doc( +Create MFCC feature files. +fbank: float, A tensor of shape a tensor of shape [audio_channels, fbank_length, fbank_feat_dim]. +sample_rate: int32, how many samples per second the source audio used. e.g. 16000, 8000. +coefficient_count: int, Number of cepstra in MFCC computation. +cepstral_lifter: float, Constant that controls scaling of MFCCs. +output: float, mfcc features, a tensor of shape [audio_channels, fbank_length, mfcc_feat_dim]. +)doc"); + +REGISTER_OP("Fbank") + .Input("spectrogram: float") + .Input("sample_rate: int32") + .Attr("upper_frequency_limit: float = 0") + .Attr("lower_frequency_limit: float = 20") + .Attr("filterbank_channel_count: int = 23") + .Output("output: float") + .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c){ + return Status::OK(); + }) +// .SetShapeFn(FbankShapeFn) + .Doc(R"doc( +Create Mel-filter bank (FBANK) feature files.( +spectrogram +: float, A tensor of shape [audio_channels, spectrogram_length, spectrogram_feat_dim]. +sample_rate: int32, how many samples per second the source audio used. e.g. 16000, 8000. +upper_frequency_limit: float, the highest frequency to use when calculating the ceptstrum. +lower_frequency_limit: float, the lowest frequency to use when calculating the ceptstrum. +filterbank_channel_count: int, resolution of the Mel bank used internally. +output: float, fbank features, a tensor of shape [audio_channels, spectrogram_length, bank_feat_dim]. +)doc"); + +// ref: https//github.com/kaldi-asr/kaldi/src/featbin/add-deltas.cc +REGISTER_OP("DeltaDelta") + .Input("features: float") + .Output("features_with_delta_delta: float") + .Attr("order: int = 2") + .Attr("window: int = 2") + .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c){ + return Status::OK(); + }) +// .SetShapeFn([](InferenceContext* c) { +// int order; +// TF_RETURN_IF_ERROR(c->GetAttr("order", &order)); +// +// ShapeHandle feat; +// TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &feat)); +// +// DimensionHandle nframe = c->Dim(feat, 0); +// DimensionHandle nfeat = c->Dim(feat, 1); +// +// DimensionHandle out_feat; +// TF_RETURN_IF_ERROR(c->Multiply(nfeat, order + 1, &out_feat)); +// +// // int input_len = c->Value(nfeat); +// // int output_len = input_len * (order + 1); +// // DimensionHandle out_feat = c->MakeDim(output_len); +// c->set_output(0, c->Matrix(nframe, out_feat)); +// return Status::OK(); +// }) + .Doc(R"doc( +Add deltas (typically to raw mfcc or plp features). +features: A matrix of shape [nframe, feat_dim]. +features_with_delta_delta: A matrix of shape [nframe, feat_dim * (order + 1)]. +order: int, order fo delta computation. +window: a int, parameter controlling window for delta computation(actual window + size for each delta order is 1 + 2*window). +)doc"); + +} // namespace delta diff --git a/athena/transform/feats/ops/py_x_ops.py b/athena/transform/feats/ops/py_x_ops.py new file mode 100644 index 00000000..018c1348 --- /dev/null +++ b/athena/transform/feats/ops/py_x_ops.py @@ -0,0 +1,34 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" python custom ops """ +import tensorflow.compat.v1 as tf +from absl import logging + +so_lib_file = tf.io.gfile.glob(tf.resource_loader.get_data_files_path() + "/x_ops*.so")[ + 0 +].split("/")[-1] +path = tf.resource_loader.get_path_to_datafile(so_lib_file) +logging.info("x_ops*.so path:{}".format(path)) + +gen_x_ops = tf.load_op_library(tf.resource_loader.get_path_to_datafile(so_lib_file)) + +spectrum = gen_x_ops.spectrum +fbank = gen_x_ops.fbank +delta_delta = gen_x_ops.delta_delta +pitch = gen_x_ops.pitch +mfcc = gen_x_ops.mfcc_dct +frame_pow = gen_x_ops.frame_pow +speed = gen_x_ops.speed diff --git a/athena/transform/feats/pitch.py b/athena/transform/feats/pitch.py new file mode 100644 index 00000000..f2d0f225 --- /dev/null +++ b/athena/transform/feats/pitch.py @@ -0,0 +1,176 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""This model extracts pitch features per frame.""" + +import tensorflow as tf + +from athena.utils.hparam import HParams +from athena.transform.feats.ops import py_x_ops +from athena.transform.feats.base_frontend import BaseFrontend + + +class Pitch(BaseFrontend): + """ + Compute pitch features of every frame in speech, return a float tensor + with size (num_frames, 2). + """ + def __init__(self, config: dict): + super().__init__(config) + + @classmethod + def params(cls, config=None): + """ + Set params. + :param config: contains nineteen optional parameters: + --delta-pitch : Smallest relative change in pitch that our algorithm + measures (float, default = 0.005) + --window_length : Frame length in seconds (float, default = 0.025) + --frame_length : Frame shift in seconds (float, default = 0.010) + --frames-per-chunk : Only relevant for offline pitch extraction (e.g. + compute-kaldi-pitch-feats), you can set it to a small + nonzero value, such as 10, for better feature + compatibility with online decoding (affects energy + normalization in the algorithm) (int, default = 0) + --lowpass-cutoff : cutoff frequency for LowPass filter (Hz). + (float, default = 1000) + --lowpass-filter-width : Integer that determines filter width of lowpass filter, + more gives sharper filter (int, default = 1) + --max-f0 : max. F0 to search for (Hz) (float, default = 400) + --max-frames-latency : Maximum number of frames of latency that we allow pitch + tracking to introduce into the feature processing + (affects output only if --frames-per-chunk > 0 and + --simulate-first-pass-online=true (int, default = 0) + --min-f0 : min. F0 to search for (Hz) (float, default = 50) + --nccf-ballast : Increasing this factor reduces NCCF for quiet frames. + (float, default = 7000) + --nccf-ballast-online : This is useful mainly for debug; it affects how the NCCF + ballast is computed. (bool, default = false) + --penalty-factor : cost factor for FO change. (float, default = 0.1) + --preemphasis-coefficient : Coefficient for use in signal preemphasis (deprecated). + (float, default = 0) + --recompute-frame : Only relevant for online pitch extraction, or for + compatibility with online pitch extraction. A + non-critical parameter; the frame at which we recompute + some of the forward pointers, after revising our + estimate of the signal energy. Relevant + if--frames-per-chunk > 0. (int, default = 500) + --resample-frequency : Frequency that we down-sample the signal to. Must be + more than twice lowpass-cutoff (float, default = 4000) + --simulate-first-pass-online : If true, compute-kaldi-pitch-feats will output features + that correspond to what an online decoder would see in + the first pass of decoding-- not the final version of + the features, which is the default. Relevant if + --frames-per-chunk > 0 (bool, default = false) + --snip-edges : If this is set to false, the incomplete frames near the + ending edge won't be snipped, so that the number of + frames is the file size divided by the frame-shift. + This makes different types of features give the same + number of frames. (bool, default = true) + --soft-min-f0 : Minimum f0, applied in soft way, must not exceed min-f0. + (float, default = 10) + --upsample-filter-width : Integer that determines filter width when upsampling + NCCF. (int, default = 5) + :return: An object of class HParams, which is a set of hyperparameters as name-value pairs. + """ + + hparams = HParams(cls=cls) + + window_length = 0.025 + frame_length = 0.010 + sample_rate = 16000 + snip_edges = True + preemph_coeff = 0.0 + min_f0 = 50.0 + max_f0 = 400.0 + soft_min_f0 = 10.0 + penalty_factor = 0.1 + lowpass_cutoff = 1000.0 + resample_freq = 4000.0 + delta_pitch = 0.005 + nccf_ballast = 7000.0 + lowpass_filter_width = 1 + upsample_filter_width = 5 + max_frames_latency = 0 + frames_per_chunk = 0 + simulate_first_pass_online = False + recompute_frame = 500 + nccf_ballast_online = False + + hparams.add_hparam("window_length", window_length) + hparams.add_hparam("frame_length", frame_length) + hparams.add_hparam("sample_rate", sample_rate) + hparams.add_hparam("snip_edges", snip_edges) + hparams.add_hparam("preemph_coeff", preemph_coeff) + hparams.add_hparam("min_f0", min_f0) + hparams.add_hparam("max_f0", max_f0) + hparams.add_hparam("soft_min_f0", soft_min_f0) + hparams.add_hparam("penalty_factor", penalty_factor) + hparams.add_hparam("lowpass_cutoff", lowpass_cutoff) + hparams.add_hparam("resample_freq", resample_freq) + hparams.add_hparam("delta_pitch", delta_pitch) + hparams.add_hparam("nccf_ballast", nccf_ballast) + hparams.add_hparam("lowpass_filter_width", lowpass_filter_width) + hparams.add_hparam("upsample_filter_width", upsample_filter_width) + hparams.add_hparam("max_frames_latency", max_frames_latency) + hparams.add_hparam("frames_per_chunk", frames_per_chunk) + hparams.add_hparam("simulate_first_pass_online", simulate_first_pass_online) + hparams.add_hparam("recompute_frame", recompute_frame) + hparams.add_hparam("nccf_ballast_online", nccf_ballast_online) + + if config is not None: + hparams.parse(config, True) + + return hparams + + def call(self, audio_data, sample_rate): + """ + Caculate picth features of audio data. + :param audio_data: the audio signal from which to compute spectrum. + Should be an (1, N) tensor. + :param sample_rate: the samplerate of the signal we working with. + :return: A float tensor of size (num_frames, 2) containing + pitch && POV features of every frame in speech. + """ + p = self.config + + with tf.name_scope('pitch'): + sample_rate = tf.cast(sample_rate, dtype=tf.int32) + pitch = py_x_ops.pitch(audio_data, + sample_rate, + window_length=p.window_length, + frame_length=p.frame_length, + snip_edges=p.snip_edges, + preemph_coeff=p.preemph_coeff, + min_f0=p.min_f0, + max_f0=p.max_f0, + soft_min_f0=p.soft_min_f0, + penalty_factor=p.penalty_factor, + lowpass_cutoff=p.lowpass_cutoff, + resample_freq=p.resample_freq, + delta_pitch=p.delta_pitch, + nccf_ballast=p.nccf_ballast, + lowpass_filter_width=p.lowpass_filter_width, + upsample_filter_width=p.upsample_filter_width, + max_frames_latency=p.max_frames_latency, + frames_per_chunk=p.frames_per_chunk, + simulate_first_pass_online=p.simulate_first_pass_online, + recompute_frame=p.recompute_frame, + nccf_ballast_online=p.nccf_ballast_online) + + return pitch + + def dim(self): + return 2 \ No newline at end of file diff --git a/athena/transform/feats/pitch_test.py b/athena/transform/feats/pitch_test.py new file mode 100644 index 00000000..aba00d78 --- /dev/null +++ b/athena/transform/feats/pitch_test.py @@ -0,0 +1,92 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The model tests pitch FE.""" + +import os +from pathlib import Path +import tensorflow as tf +from tensorflow.python.framework.ops import disable_eager_execution +from athena.transform.feats.read_wav import ReadWav +from athena.transform.feats.pitch import Pitch + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + + +class SpectrumTest(tf.test.TestCase): + """ + Pitch extraction test. + """ + def test_spectrum(self): + wav_path_16k = str( + Path(os.environ["MAIN_ROOT"]).joinpath("examples/sm1_cln.wav") + ) + wav_path_8k = str( + Path(os.environ["MAIN_ROOT"]).joinpath("examples/english.wav") + ) + + with self.session(): + for wav_file in [wav_path_16k]: + read_wav = ReadWav.params().instantiate() + input_data, sample_rate = read_wav(wav_file) + + pitch = Pitch.params( + {"window_length": 0.025, "soft_min_f0": 10.0} + ).instantiate() + pitch_test = pitch(input_data, sample_rate) + + if tf.executing_eagerly(): + self.assertEqual(tf.rank(pitch_test).numpy(), 2) + else: + self.assertEqual(tf.rank(pitch_test).eval(), 2) + + output_true = [ + [-0.1366025, 143.8855], + [-0.0226383, 143.8855], + [-0.08464742, 143.8855], + [-0.08458386, 143.8855], + [-0.1208689, 143.8855], + ] + + if wav_file == wav_path_16k: + if tf.executing_eagerly(): + print("Transform: ", pitch_test.numpy()[0:5, :]) + print("kaldi:", output_true) + self.assertAllClose( + pitch_test.numpy()[0:5, :], + output_true, + rtol=1e-05, + atol=1e-05, + ) + else: + print("Transform: ", pitch_test.eval()) + print("kaldi:", output_true) + self.assertAllClose( + pitch_test.eval()[0:5, :], + output_true, + rtol=1e-05, + atol=1e-05, + ) + + +if __name__ == "__main__": + + is_eager = True + if not is_eager: + disable_eager_execution() + else: + if tf.__version__ < "2.0.0": + tf.compat.v1.enable_eager_execution() + tf.test.main() diff --git a/athena/transform/feats/read_wav.py b/athena/transform/feats/read_wav.py new file mode 100644 index 00000000..df0bcc11 --- /dev/null +++ b/athena/transform/feats/read_wav.py @@ -0,0 +1,84 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The model reads audio sample from wav file.""" + +import tensorflow as tf +from athena.utils.hparam import HParams +from athena.transform.feats.base_frontend import BaseFrontend +from athena.transform.feats.ops import py_x_ops + + +class ReadWav(BaseFrontend): + """ + Read audio sample from wav file, return sample data and sample rate. + """ + def __init__(self, config: dict): + super().__init__(config) + + @classmethod + def params(cls, config=None): + """ + Set params. + :param config: contains one optional parameters: audio_channels(int, default=1). + :return: An object of class HParams, which is a set of hyperparameters as + name-value pairs. + """ + audio_channels = 1 + + hparams = HParams(cls=cls) + hparams.add_hparam('type', 'ReadWav') + hparams.add_hparam('audio_channels', audio_channels) + + if config is not None: + hparams.parse(config, True) + + return hparams + + def call(self, wavfile, speed=1.0): + """ + Get audio data and sample rate from a wavfile. + :param wavfile: filepath of wav + speed: Speed of sample channels wanted (float, default=1.0) + :return: 2 values. The first is a Tensor of audio data. + The second return value isthe sample rate of the input wav + file, which is a tensor with float dtype. + """ + p = self.config + contents = tf.io.read_file(wavfile) + audio_data, sample_rate = tf.compat.v1.audio.decode_wav( + contents, desired_channels=p.audio_channels) + if (speed == 1.0): + return tf.squeeze(audio_data * 32768, axis=-1), tf.cast(sample_rate, dtype=tf.int32) + else: + resample_rate = tf.cast(sample_rate, dtype=tf.float32) * tf.cast(1.0 / speed, dtype=tf.float32) + speed_data = py_x_ops.speed(tf.squeeze(audio_data * 32768, axis=-1), + tf.cast(sample_rate, dtype=tf.int32), + tf.cast(resample_rate, dtype=tf.int32), + lowpass_filter_width=5) + return tf.squeeze(speed_data), tf.cast(sample_rate, dtype=tf.int32) + + +def read_wav(wavfile, audio_channels=1): + """ read wav from file + args: audio_channels = 1 + returns: tf.squeeze(audio_data * 32768, axis=-1), tf.cast(sample_rate, dtype=tf.int32) + """ + contents = tf.io.read_file(wavfile) + audio_data, sample_rate = tf.compat.v1.audio.decode_wav( + contents, desired_channels=audio_channels + ) + + return tf.squeeze(audio_data * 32768, axis=-1), tf.cast(sample_rate, dtype=tf.int32) diff --git a/athena/transform/feats/read_wav_test.py b/athena/transform/feats/read_wav_test.py new file mode 100644 index 00000000..bf3b7e97 --- /dev/null +++ b/athena/transform/feats/read_wav_test.py @@ -0,0 +1,57 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The model tests OP of read_wav """ + +import os +from pathlib import Path +import librosa +import tensorflow as tf +from tensorflow.python.framework.ops import disable_eager_execution +from athena.transform.feats.read_wav import ReadWav + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + +class ReadWavTest(tf.test.TestCase): + """ + ReadWav OP test. + """ + def test_read_wav(self): + wav_path = str(Path(os.environ['MAIN_ROOT']).joinpath('examples/sm1_cln.wav')) + + with self.session(): + speed = 0.9 + read_wav = ReadWav.params().instantiate() + input_data, sample_rate = read_wav(wav_path, speed) + + audio_data_true, sample_rate_true = librosa.load(wav_path, sr=16000) + if (speed == 1.0): + if tf.executing_eagerly(): + self.assertAllClose(input_data.numpy() / 32768, audio_data_true) + self.assertAllClose(sample_rate.numpy(), sample_rate_true) + else: + self.assertAllClose(input_data.eval() / 32768, audio_data_true) + self.assertAllClose(sample_rate.eval(), sample_rate_true) + + +if __name__ == '__main__': + + is_eager = False + if not is_eager: + disable_eager_execution() + else: + if tf.__version__ < '2.0.0': + tf.compat.v1.enable_eager_execution() + tf.test.main() diff --git a/athena/transform/feats/spectrum.py b/athena/transform/feats/spectrum.py new file mode 100644 index 00000000..e0b98f59 --- /dev/null +++ b/athena/transform/feats/spectrum.py @@ -0,0 +1,133 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""This model extracts spetrum features per frame.""" + +import math +import tensorflow as tf + +from athena.utils.hparam import HParams +from athena.transform.feats.ops import py_x_ops +from athena.transform.feats.base_frontend import BaseFrontend +from athena.transform.feats.cmvn import CMVN + + +class Spectrum(BaseFrontend): + """ + Compute spectrum features of every frame in speech, return a float tensor + with size (num_frames, num_frequencies). + """ + def __init__(self, config: dict): + super().__init__(config) + self.cmvn = CMVN(config) + + @classmethod + def params(cls, config=None): + """ + Set params. + :param config: contains nine optional parameters： + --window_length : Window length in seconds. (float, default = 0.025) + --frame_length : Hop length in seconds. (float, default = 0.010) + --snip_edges : If 1, the last frame (shorter than window_length) + will be cutoff. If 2, 1 // 2 frame_length data will + be padded to data. (int, default = 1) + ---raw_energy : If 1, compute frame energy before preemphasis and windowing. + If 2, compute frame energy after preemphasis and windowing. + (int, default = 1) + --preEph_coeff : Coefficient for use in frame-signal preemphasis. + (float, default = 0.97) + --window_type : Type of window ("hamm"|"hann"|"povey"|"rect"|"blac"|"tria"). + (string, default = "povey") + --remove_dc_offset : Subtract mean from waveform on each frame. + (bool, default = true) + --is_fbank : If true, compute power spetrum without frame energy. + If false, using the frame energy instead of the square of the + constant component of the signal. (bool, default = false) + --output_type : If 1, return power spectrum. If 2, return log-power spectrum. + (int, default = 2) + --dither : Dithering constant (0.0 means no dither). + (float, default = 1) [add robust to training] + :return: An object of class HParams, which is a set of hyperparameters as name-value pairs. + """ + + window_length = 0.025 + frame_length = 0.010 + output_type = 2 + snip_edges = 1 + raw_energy = 1 + preEph_coeff = 0.97 + window_type = "povey" + remove_dc_offset = True + is_fbank = False + dither = 0.0 + + hparams = HParams(cls=cls) + hparams.add_hparam("window_length", window_length) + hparams.add_hparam("frame_length", frame_length) + hparams.add_hparam("output_type", output_type) + hparams.add_hparam("snip_edges", snip_edges) + hparams.add_hparam("raw_energy", raw_energy) + hparams.add_hparam("preEph_coeff", preEph_coeff) + hparams.add_hparam("window_type", window_type) + hparams.add_hparam("remove_dc_offset", remove_dc_offset) + hparams.add_hparam("is_fbank", is_fbank) + hparams.add_hparam("dither", dither) + + # cmvn + hparams.append(CMVN.params()) + + if config is not None: + hparams.parse(config, True) + hparams.type = "Spectrum" + + return hparams + + def call(self, audio_data, sample_rate=None): + """ + Caculate power spectrum or log power spectrum of audio data. + :param audio_data: the audio signal from which to compute spectrum. + Should be an (1, N) tensor. + :param sample_rate: [option]the samplerate of the signal we working with, default is 16kHz. + :return: A float tensor of size (num_frames, num_frequencies) containing power + spectrum (output_type=1) or log power spectrum (output_type=2) + of every frame in speech. + """ + + p = self.config + with tf.name_scope("spectrum"): + + sample_rate = tf.cast(sample_rate, dtype=float) + spectrum = py_x_ops.spectrum( + audio_data, + sample_rate, + window_length=p.window_length, + frame_length=p.frame_length, + output_type=p.output_type, + snip_edges=p.snip_edges, + raw_energy=p.raw_energy, + preEph_coeff=p.preEph_coeff, + window_type=p.window_type, + remove_dc_offset=p.remove_dc_offset, + is_fbank=p.is_fbank, + dither=p.dither, + ) + + if p.type == "Spectrum": + spectrum = self.cmvn(spectrum) + + return spectrum + + def dim(self): + return 1 diff --git a/athena/transform/feats/spectrum_test.py b/athena/transform/feats/spectrum_test.py new file mode 100644 index 00000000..cae61e2a --- /dev/null +++ b/athena/transform/feats/spectrum_test.py @@ -0,0 +1,87 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The model tests spectrum FE.""" + +import os +from pathlib import Path +import numpy as np +import tensorflow as tf +from tensorflow.python.framework.ops import disable_eager_execution +from athena.transform.feats.read_wav import ReadWav +from athena.transform.feats.spectrum import Spectrum + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + + +class SpectrumTest(tf.test.TestCase): + ''' + Spectum extraction test. + ''' + def test_spectrum(self): + wav_path_16k = str( + Path(os.environ["MAIN_ROOT"]).joinpath("examples/sm1_cln.wav") + ) + wav_path_8k = str( + Path(os.environ["MAIN_ROOT"]).joinpath("examples/english.wav") + ) + + with self.session(): + for wav_file in [wav_path_8k, wav_path_16k]: + read_wav = ReadWav.params().instantiate() + input_data, sample_rate = read_wav(wav_file) + + spectrum = Spectrum.params( + {"window_length": 0.025, "dither": 0.0} + ).instantiate() + spectrum_test = spectrum(input_data, sample_rate) + + output_true = np.array( + [ + [9.819611, 2.84503, 3.660894, 2.7779, 1.212233], + [9.328745, 2.553949, 3.276319, 3.000918, 2.499342], + ] + ) + if tf.executing_eagerly(): + self.assertEqual(tf.rank(spectrum_test).numpy(), 2) + else: + self.assertEqual(tf.rank(spectrum_test).eval(), 2) + + if wav_file == wav_path_16k: + if tf.executing_eagerly(): + self.assertAllClose( + spectrum_test.numpy()[0:2, 0:5], + output_true, + rtol=1e-05, + atol=1e-05, + ) + else: + self.assertAllClose( + spectrum_test.eval()[0:2, 0:5], + output_true, + rtol=1e-05, + atol=1e-05, + ) + + +if __name__ == "__main__": + + is_eager = True + if not is_eager: + disable_eager_execution() + else: + if tf.__version__ < "2.0.0": + tf.compat.v1.enable_eager_execution() + tf.test.main() diff --git a/athena/transform/feats/write_wav.py b/athena/transform/feats/write_wav.py new file mode 100644 index 00000000..81eea449 --- /dev/null +++ b/athena/transform/feats/write_wav.py @@ -0,0 +1,67 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The model write audio sample to wav file.""" + +import tensorflow as tf +from athena.utils.hparam import HParams +from athena.transform.feats.base_frontend import BaseFrontend + + +class WriteWav(BaseFrontend): + """ + Encode audio data (input) using sample rate (input), + return a write wav opration. + """ + + def __init__(self, config: dict): + super().__init__(config) + + @classmethod + def params(cls, config=None): + """ + Set params. + :param config: contains one optional parameters:sample_rate(int, default=16000). + :return: An object of class HParams, which is a set of hyperparameters as + name-value pairs. + """ + + sample_rate = 16000 + + hparams = HParams(cls=cls) + hparams.add_hparam('sample_rate', sample_rate) + + if config is not None: + hparams.override_from_dict(config) + + return hparams + + def call(self, filename, audio_data, sample_rate): + """ + Write wav using audio_data[tensor]. + :param filename: filepath of wav. + :param audio_data: a tensor containing data of a wav. + :param sample_rate: the samplerate of the signal we working with. + :return: write wav opration. + """ + filename = tf.constant(filename) + + with tf.name_scope('writewav'): + audio_data = tf.cast(audio_data, dtype=tf.float32) + contents = tf.audio.encode_wav( + tf.expand_dims(audio_data, 1), tf.cast(sample_rate, dtype=tf.int32)) + w_op = tf.io.write_file(filename, contents) + + return w_op diff --git a/athena/transform/feats/write_wav_test.py b/athena/transform/feats/write_wav_test.py new file mode 100644 index 00000000..20d14fcb --- /dev/null +++ b/athena/transform/feats/write_wav_test.py @@ -0,0 +1,55 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The model tests WriteWav OP.""" + +import os + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" +from pathlib import Path +import librosa +import tensorflow as tf +from tensorflow.python.framework.ops import disable_eager_execution +from athena.transform.feats.read_wav import ReadWav +from athena.transform.feats.write_wav import WriteWav + + +class WriteWavTest(tf.test.TestCase): + """ + WriteWav OP test. + """ + def test_write_wav(self): + wav_path = str(Path(os.environ["MAIN_ROOT"]).joinpath("examples/sm1_cln.wav")) + + with self.cached_session() as sess: + config = {"speed": 1.1} + read_wav = ReadWav.params(config).instantiate() + input_data, sample_rate = read_wav(wav_path) + write_wav = WriteWav.params().instantiate() + new_path = str( + Path(os.environ["MAIN_ROOT"]).joinpath("examples/sm1_cln_resample.wav") + ) + writewav_op = write_wav(new_path, input_data / 32768, sample_rate) + sess.run(writewav_op) + + +if __name__ == "__main__": + is_eager = True + if not is_eager: + disable_eager_execution() + else: + if tf.__version__ < "2.0.0": + tf.compat.v1.enable_eager_execution() + tf.test.main() diff --git a/athena/utils/__init__.py b/athena/utils/__init__.py new file mode 100644 index 00000000..559b770f --- /dev/null +++ b/athena/utils/__init__.py @@ -0,0 +1,16 @@ +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" utils """ diff --git a/athena/utils/checkpoint.py b/athena/utils/checkpoint.py new file mode 100644 index 00000000..a4d9d9ae --- /dev/null +++ b/athena/utils/checkpoint.py @@ -0,0 +1,76 @@ +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support eager mode +# pylint: disable=invalid-name +r""" check point manager """ +import os +import tensorflow as tf +from absl import logging +import numpy as np + +class Checkpoint(tf.train.Checkpoint): + """ A wrapper for Tensorflow checkpoint + + Args: + checkpoint_directory: the directory for checkpoint + summary_directory: the directory for summary used in Tensorboard + __init__ provide the optimizer and model + __call__ save the model + + Example: + transformer = SpeechTransformer(target_vocab_size=dataset_builder.target_dim) + optimizer = tf.keras.optimizers.Adam() + ckpt = Checkpoint(checkpoint_directory='./train', summary_directory='./event', + transformer=transformer, optimizer=optimizer) + solver = BaseSolver(transformer) + for epoch in dataset: + ckpt() + """ + + def __init__(self, checkpoint_directory=None, **kwargs): + super().__init__(**kwargs) + self.best_loss = np.inf + if checkpoint_directory is None: + checkpoint_directory = os.path.join(os.path.expanduser("~"), ".athena") + self.checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + self.checkpoint_directory = checkpoint_directory + logging.info("trying to restore from : %s" % checkpoint_directory) + # load from checkpoint if previous models exist in checkpoint dir + self.restore(tf.train.latest_checkpoint(checkpoint_directory)) + + def _compare_and_save_best(self, loss, save_path): + """ compare and save the best model in best_loss """ + if loss is None: + return + if loss < self.best_loss: + self.best_loss = loss + with open(os.path.join(self.checkpoint_directory, 'best_loss'), 'w') as wf: + checkpoint = save_path.split('/')[-1] + wf.write('model_checkpoint_path: "%s"' % checkpoint) + + def __call__(self, loss=None): + logging.info("saving model in :%s" % self.checkpoint_prefix) + save_path = self.save(file_prefix=self.checkpoint_prefix) + self._compare_and_save_best(loss, save_path) + + def restore_from_best(self): + """ restore from the best model """ + self.restore( + tf.train.latest_checkpoint( + self.checkpoint_directory, + latest_filename='best_loss' + ) + ) \ No newline at end of file diff --git a/athena/utils/data_queue.py b/athena/utils/data_queue.py new file mode 100644 index 00000000..7f8789fb --- /dev/null +++ b/athena/utils/data_queue.py @@ -0,0 +1,104 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=missing-function-docstring, invalid-name +""" data queue for multi thread """ +import time +import threading +import queue + + +class DataQueue: + """Queue for data prefetching + args: + generator(generator): instance of generator which feed data + capacity(int): maximum data to prefetch + num_threads(int): control concurrency, only take effect when do preprocessing + wait_time(float): time to sleep when queue is full + """ + + def __init__(self, generator, capacity=20, num_threads=4, max_index=10000, wait_time=0.0001): + self.generator = generator + self.capacity = capacity + self.wait_time = wait_time + self.queue = queue.Queue() + self.index = 0 + self.max_index = max_index + + self._stop = threading.Event() + self._lock = threading.Lock() + + self.threads = [ + threading.Thread(target=self.generator_task) for _ in range(num_threads) + ] + + for t in self.threads: + t.setDaemon(True) + t.start() + + def __del__(self): + self.stop() + + def get(self): + return self.queue.get() + + def stop(self): + self._stop.set() + + def generator_task(self): + """Enqueue batch data""" + while not self._stop.is_set(): + try: + if self.index >= self.max_index: + continue + batch = self.generator(self.index) + self._lock.acquire() + if self.queue.qsize() < self.capacity: + try: + self.index = self.index + 1 + except ValueError as e: + print(e) + self._lock.release() + continue + self.queue.put(batch) + self._lock.release() + else: + self._lock.release() + time.sleep(self.wait_time) + except Exception as e: + print(e) + self._stop.set() + raise + + +def test(): + """ + Test data queue. + Excpet return: + epoch: %d, nb_batch: %d: finish. + """ + + def generator(i): + return i + + train_queue = DataQueue(generator, capacity=8, num_threads=4) + for _ in range(92): + print(train_queue.get()) + train_queue.stop() + + +if __name__ == "__main__": + test() diff --git a/athena/utils/hparam.py b/athena/utils/hparam.py new file mode 100644 index 00000000..7ad86bac --- /dev/null +++ b/athena/utils/hparam.py @@ -0,0 +1,695 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# Forked with minor changes from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/training/python/training/hparam.py pylint: disable=line-too-long +"""Hyperparameter values.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import numbers +import re +import six + +# Define the regular expression for parsing a single clause of the input +# (delimited by commas). A legal clause looks like: +# []? = +# where is either a single token or [] enclosed list of tokens. +# For example: "var[1] = a" or "x = [1,2,3]" +PARAM_RE = re.compile( + r""" + (?P[a-zA-Z][\w\.]*) # variable name: "var" or "x" + (\[\s*(?P\d+)\s*\])? # (optional) index: "1" or None + \s*=\s* + ((?P[^,\[]*) # single value: "a" or None + | + \[(?P[^\]]*)\]) # list of values: None or "1,2,3" + ($|,\s*)""", + re.VERBOSE, +) + + +def _parse_fail(name, var_type, value, values): + """Helper function for raising a value error for bad assignment.""" + raise ValueError( + "Could not parse hparam '%s' of type '%s' with value '%s' in %s" + % (name, var_type.__name__, value, values) + ) + + +def _reuse_fail(name, values): + """Helper function for raising a value error for reuse of name.""" + raise ValueError("Multiple assignments to variable '%s' in %s" % (name, values)) + + +def _process_scalar_value(name, parse_fn, var_type, m_dict, values, results_dictionary): + """Update results_dictionary with a scalar value. + + Used to update the results_dictionary to be returned by parse_values when + encountering a clause with a scalar RHS (e.g. "s=5" or "arr[0]=5".) + + Mutates results_dictionary. + + Args: + name: Name of variable in assignment ("s" or "arr"). + parse_fn: Function for parsing the actual value. + var_type: Type of named variable. + m_dict: Dictionary constructed from regex parsing. + m_dict['val']: RHS value (scalar) + m_dict['index']: List index value (or None) + values: Full expression being parsed + results_dictionary: The dictionary being updated for return by the parsing function. + + Raises: + ValueError: If the name has already been used. + """ + try: + parsed_value = parse_fn(m_dict["val"]) + except ValueError: + _parse_fail(name, var_type, m_dict["val"], values) + + # If no index is provided + if not m_dict["index"]: + if name in results_dictionary: + _reuse_fail(name, values) + results_dictionary[name] = parsed_value + else: + if name in results_dictionary: + # The name has already been used as a scalar, then it + # will be in this dictionary and map to a non-dictionary. + if not isinstance(results_dictionary.get(name), dict): + _reuse_fail(name, values) + else: + results_dictionary[name] = {} + + index = int(m_dict["index"]) + # Make sure the index position hasn't already been assigned a value. + if index in results_dictionary[name]: + _reuse_fail("{}[{}]".format(name, index), values) + results_dictionary[name][index] = parsed_value + + +def _process_list_value(name, parse_fn, var_type, m_dict, values, results_dictionary): + """Update results_dictionary from a list of values. + + Used to update results_dictionary to be returned by parse_values when + encountering a clause with a list RHS (e.g. "arr=[1,2,3]".) + + Mutates results_dictionary. + + Args: + name: Name of variable in assignment ("arr"). + parse_fn: Function for parsing individual values. + var_type: Type of named variable. + m_dict: Dictionary constructed from regex parsing. + m_dict['val']: RHS value (scalar) + values: Full expression being parsed + results_dictionary: The dictionary being updated for return by the parsing function. + + Raises: + ValueError: If the name has an index or the values cannot be parsed. + """ + if m_dict["index"] is not None: + raise ValueError("Assignment of a list to a list index.") + elements = filter(None, re.split("[ ,]", m_dict["vals"])) + # Make sure the name hasn't already been assigned a value + if name in results_dictionary: + raise _reuse_fail(name, values) + try: + results_dictionary[name] = [parse_fn(e) for e in elements] + except ValueError: + _parse_fail(name, var_type, m_dict["vals"], values) + + +def _cast_to_type_if_compatible(name, param_type, value): + """Cast hparam to the provided type, if compatible. + + Args: + name: Name of the hparam to be cast. + param_type: The type of the hparam. + value: The value to be cast, if compatible. + + Returns: + The result of casting `value` to `param_type`. + + Raises: + ValueError: If the type of `value` is not compatible with param_type. + * If `param_type` is a string type, but `value` is not. + * If `param_type` is a boolean, but `value` is not, or vice versa. + * If `param_type` is an integer type, but `value` is not. + * If `param_type` is a float type, but `value` is not a numeric type. + """ + fail_msg = "Could not cast hparam '%s' of type '%s' from value %r" % ( + name, + param_type, + value, + ) + + # Some callers use None, for which we can't do any casting/checking. :( + if issubclass(param_type, type(None)): + return value + + # Avoid converting a non-string type to a string. + if issubclass(param_type, (six.string_types, six.binary_type)) and not isinstance( + value, (six.string_types, six.binary_type) + ): + raise ValueError(fail_msg) + + # Avoid converting a number or string type to a boolean or vice versa. + if issubclass(param_type, bool) != isinstance(value, bool): + raise ValueError(fail_msg) + + # Avoid converting float to an integer (the reverse is fine). + if issubclass(param_type, numbers.Integral) and not isinstance( + value, numbers.Integral + ): + raise ValueError(fail_msg) + + # Avoid converting a non-numeric type to a numeric type. + if issubclass(param_type, numbers.Number) and not isinstance(value, numbers.Number): + raise ValueError(fail_msg) + + return param_type(value) + + +def parse_values(values, type_map, ignore_unknown=False): + """Parses hyperparameter values from a string into a python map. + + `values` is a string containing comma-separated `name=value` pairs. + For each pair, the value of the hyperparameter named `name` is set to + `value`. + + If a hyperparameter name appears multiple times in `values`, a ValueError + is raised (e.g. 'a=1,a=2', 'a[1]=1,a[1]=2'). + + If a hyperparameter name in both an index assignment and scalar assignment, + a ValueError is raised. (e.g. 'a=[1,2,3],a[0] = 1'). + + The hyperparameter name may contain '.' symbols, which will result in an + attribute name that is only accessible through the getattr and setattr + functions. (And must be first explicit added through add_hparam.) + + WARNING: Use of '.' in your variable names is allowed, but is not well + supported and not recommended. + + The `value` in `name=value` must follows the syntax according to the + type of the parameter: + + * Scalar integer: A Python-parsable integer point value. E.g.: 1, + 100, -12. + * Scalar float: A Python-parsable floating point value. E.g.: 1.0, + -.54e89. + * Boolean: Either true or false. + * Scalar string: A non-empty sequence of characters, excluding comma, + spaces, and square brackets. E.g.: foo, bar_1. + * List: A comma separated list of scalar values of the parameter type + enclosed in square brackets. E.g.: [1,2,3], [1.0,1e-12], [high,low]. + + When index assignment is used, the corresponding type_map key should be the + list name. E.g. for "arr[1]=0" the type_map must have the key "arr" (not + "arr[1]"). + + Args: + values: String. Comma separated list of `name=value` pairs where + 'value' must follow the syntax described above. + type_map: A dictionary mapping hyperparameter names to types. Note every + parameter name in values must be a key in type_map. The values must + conform to the types indicated, where a value V is said to conform to a + type T if either V has type T, or V is a list of elements of type T. + Hence, for a multidimensional parameter 'x' taking float values, + 'x=[0.1,0.2]' will parse successfully if type_map['x'] = float. + ignore_unknown: Bool. Whether values that are missing a type in type_map + should be ignored. If set to True, a ValueError will not be raised for + unknown hyperparameter type. + + Returns: + A python map mapping each name to either: + * A scalar value. + * A list of scalar values. + * A dictionary mapping index numbers to scalar values. + (e.g. "x=5,L=[1,2],arr[1]=3" results in {'x':5,'L':[1,2],'arr':{1:3}}") + + Raises: + ValueError: If there is a problem with input. + * If `values` cannot be parsed. + * If a list is assigned to a list index (e.g. 'a[1] = [1,2,3]'). + * If the same rvalue is assigned two different values (e.g. 'a=1,a=2', + 'a[1]=1,a[1]=2', or 'a=1,a=[1]') + """ + results_dictionary = {} + pos = 0 + while pos < len(values): + m = PARAM_RE.match(values, pos) + if not m: + raise ValueError("Malformed hyperparameter value: %s" % values[pos:]) + # Check that there is a comma between parameters and move past it. + pos = m.end() + # Parse the values. + m_dict = m.groupdict() + name = m_dict["name"] + if name not in type_map: + if ignore_unknown: + continue + raise ValueError("Unknown hyperparameter type for %s" % name) + type_ = type_map[name] + + # Set up correct parsing function (depending on whether type_ is a bool) + if type_ == bool: + + def parse_bool(value): + if value in ["true", "True"]: + return True + elif value in ["false", "False"]: + return False + else: + try: + return bool(int(value)) + except ValueError: + _parse_fail(name, type_, value, values) + + parse = parse_bool + else: + parse = type_ + + # If a singe value is provided + if m_dict["val"] is not None: + _process_scalar_value( + name, parse, type_, m_dict, values, results_dictionary + ) + + # If the assigned value is a list: + elif m_dict["vals"] is not None: + _process_list_value(name, parse, type_, m_dict, values, results_dictionary) + + else: # Not assigned a list or value + _parse_fail(name, type_, "", values) + + return results_dictionary + + +class HParams(object): + """Class to hold a set of hyperparameters as name-value pairs. + + A `HParams` object holds hyperparameters used to build and train a model, + such as the number of hidden units in a neural net layer or the learning rate + to use when training. + + You first create a `HParams` object by specifying the names and values of the + hyperparameters. + + To make them easily accessible the parameter names are added as direct + attributes of the class. A typical usage is as follows: + + ```python + # Create a HParams object specifying names and values of the model + # hyperparameters: + hparams = HParams(learning_rate=0.1, num_hidden_units=100) + + # The hyperparameter are available as attributes of the HParams object: + hparams.learning_rate ==> 0.1 + hparams.num_hidden_units ==> 100 + ``` + + Hyperparameters have type, which is inferred from the type of their value + passed at construction type. The currently supported types are: integer, + float, boolean, string, and list of integer, float, boolean, or string. + + You can override hyperparameter values by calling the + [`parse()`](#HParams.parse) method, passing a string of comma separated + `name=value` pairs. This is intended to make it possible to override + any hyperparameter values from a single command-line flag to which + the user passes 'hyper-param=value' pairs. It avoids having to define + one flag for each hyperparameter. + + The syntax expected for each value depends on the type of the parameter. + See `parse()` for a description of the syntax. + + Example: + + ```python + # Define a command line flag to pass name=value pairs. + # For example using argparse: + import argparse + parser = argparse.ArgumentParser(description='Train my model.') + parser.add_argument('--hparams', type=str, + help='Comma separated list of "name=value" pairs.') + args = parser.parse_args() + ... + def my_program(): + # Create a HParams object specifying the names and values of the + # model hyperparameters: + hparams = tf.HParams(learning_rate=0.1, num_hidden_units=100, + activations=['relu', 'tanh']) + + # Override hyperparameters values by parsing the command line + hparams.parse(args.hparams) + + # If the user passed `--hparams=learning_rate=0.3` on the command line + # then 'hparams' has the following attributes: + hparams.learning_rate ==> 0.3 + hparams.num_hidden_units ==> 100 + hparams.activations ==> ['relu', 'tanh'] + + # If the hyperparameters are in json format use parse_json: + hparams.parse_json('{"learning_rate": 0.3, "activations": "relu"}') + ``` + """ + + _HAS_DYNAMIC_ATTRIBUTES = True # Required for pytype checks. + + def __init__(self, model_structure=None, **kwargs): + """Create an instance of `HParams` from keyword arguments. + + The keyword arguments specify name-values pairs for the hyperparameters. + The parameter types are inferred from the type of the values passed. + + The parameter names are added as attributes of `HParams` object, so they + can be accessed directly with the dot notation `hparams._name_`. + + Example: + + ```python + # Define 3 hyperparameters: 'learning_rate' is a float parameter, + # 'num_hidden_units' an integer parameter, and 'activation' a string + # parameter. + hparams = tf.HParams( + learning_rate=0.1, num_hidden_units=100, activation='relu') + + hparams.activation ==> 'relu' + ``` + + Note that a few names are reserved and cannot be used as hyperparameter + names. If you use one of the reserved name the constructor raises a + `ValueError`. + + Args: + model_structure: An instance of ModelStructure, defining the feature + crosses to be used in the Trial. + **kwargs: Key-value pairs where the key is the hyperparameter name and + the value is the value for the parameter. + + Raises: + ValueError: If both `hparam_def` and initialization values are provided, + or if one of the arguments is invalid. + + """ + # Register the hyperparameters and their type in _hparam_types. + # This simplifies the implementation of parse(). + # _hparam_types maps the parameter name to a tuple (type, bool). + # The type value is the type of the parameter for scalar hyperparameters, + # or the type of the list elements for multidimensional hyperparameters. + # The bool value is True if the value is a list, False otherwise. + self._hparam_types = {} + self._model_structure = model_structure + for name, value in six.iteritems(kwargs): + self.add_hparam(name, value) + + def add_hparam(self, name, value): + """Adds {name, value} pair to hyperparameters. + + Args: + name: Name of the hyperparameter. + value: Value of the hyperparameter. Can be one of the following types: + int, float, string, int list, float list, or string list. + + Raises: + ValueError: if one of the arguments is invalid. + """ + # Keys in kwargs are unique, but 'name' could the name of a pre-existing + # attribute of this object. In that case we refuse to use it as a + # hyperparameter name. + if getattr(self, name, None) is not None: + raise ValueError("Hyperparameter name is reserved: %s" % name) + if isinstance(value, (list, tuple)): + if not value: + raise ValueError( + "Multi-valued hyperparameters cannot be empty: %s" % name + ) + self._hparam_types[name] = (type(value[0]), True) + else: + self._hparam_types[name] = (type(value), False) + setattr(self, name, value) + + def set_hparam(self, name, value): + """Set the value of an existing hyperparameter. + + This function verifies that the type of the value matches the type of the + existing hyperparameter. + + Args: + name: Name of the hyperparameter. + value: New value of the hyperparameter. + + Raises: + KeyError: If the hyperparameter doesn't exist. + ValueError: If there is a type mismatch. + """ + param_type, is_list = self._hparam_types[name] + if isinstance(value, list): + if not is_list: + raise ValueError( + "Must not pass a list for single-valued parameter: %s" % name + ) + setattr( + self, + name, + [_cast_to_type_if_compatible(name, param_type, v) for v in value], + ) + else: + if is_list: + raise ValueError( + "Must pass a list for multi-valued parameter: %s." % name + ) + setattr(self, name, _cast_to_type_if_compatible(name, param_type, value)) + + def del_hparam(self, name): + """Removes the hyperparameter with key 'name'. + + Does nothing if it isn't present. + + Args: + name: Name of the hyperparameter. + """ + if hasattr(self, name): + delattr(self, name) + del self._hparam_types[name] + + def parse(self, values, ignore_unknown=False): + """Override existing hyperparameter values, parsing new values from a string. + + See parse_values for more detail on the allowed format for values. + + Args: + values: String. Comma separated list of `name=value` pairs where 'value' + must follow the syntax described above. + + Returns: + The `HParams` instance. + + Raises: + ValueError: If `values` cannot be parsed or a hyperparameter in `values` + doesn't exist. + """ + type_map = {} + for name, t in self._hparam_types.items(): + param_type, _ = t + type_map[name] = param_type + + if type(values) is dict: + values_str = [] + for key, val in values.items(): + values_str.append(str(key) + "=" + str(val)) + values = ",".join(values_str) + + values_map = parse_values(values, type_map, ignore_unknown) + return self.override_from_dict(values_map) + + def override_from_dict(self, values_dict): + """Override existing hyperparameter values, parsing new values from a dictionary. + + Args: + values_dict: Dictionary of name:value pairs. + + Returns: + The `HParams` instance. + + Raises: + KeyError: If a hyperparameter in `values_dict` doesn't exist. + ValueError: If `values_dict` cannot be parsed. + """ + for name, value in values_dict.items(): + self.set_hparam(name, value) + return self + + def set_model_structure(self, model_structure): + self._model_structure = model_structure + + def get_model_structure(self): + return self._model_structure + + def to_json(self, indent=None, separators=None, sort_keys=False): + """Serializes the hyperparameters into JSON. + + Args: + indent: If a non-negative integer, JSON array elements and object members + will be pretty-printed with that indent level. An indent level of 0, or + negative, will only insert newlines. `None` (the default) selects the + most compact representation. + separators: Optional `(item_separator, key_separator)` tuple. Default is + `(', ', ': ')`. + sort_keys: If `True`, the output dictionaries will be sorted by key. + + Returns: + A JSON string. + """ + + def remove_callables(x): + """Omit callable elements from input with arbitrary nesting.""" + if isinstance(x, dict): + return { + k: remove_callables(v) + for k, v in six.iteritems(x) + if not callable(v) + } + elif isinstance(x, list): + return [remove_callables(i) for i in x if not callable(i)] + return x + + return json.dumps( + remove_callables(self.values()), + indent=indent, + separators=separators, + sort_keys=sort_keys, + ) + + def parse_json(self, values_json): + """Override existing hyperparameter values, parsing new values from a json object. + + Args: + values_json: String containing a json object of name:value pairs. + + Returns: + The `HParams` instance. + + Raises: + KeyError: If a hyperparameter in `values_json` doesn't exist. + ValueError: If `values_json` cannot be parsed. + """ + values_map = json.loads(values_json) + return self.override_from_dict(values_map) + + def values(self): + """Return the hyperparameter values as a Python dictionary. + + Returns: + A dictionary with hyperparameter names as keys. The values are the + hyperparameter values. + """ + return {n: getattr(self, n) for n in self._hparam_types.keys()} + + def get(self, key, default=None): + """Returns the value of `key` if it exists, else `default`.""" + if key in self._hparam_types: + # Ensure that default is compatible with the parameter type. + if default is not None: + param_type, is_param_list = self._hparam_types[key] + type_str = "list<%s>" % param_type if is_param_list else str(param_type) + fail_msg = ( + "Hparam '%s' of type '%s' is incompatible with " + "default=%s" % (key, type_str, default) + ) + + is_default_list = isinstance(default, list) + if is_param_list != is_default_list: + raise ValueError(fail_msg) + + try: + if is_default_list: + for value in default: + _cast_to_type_if_compatible(key, param_type, value) + else: + _cast_to_type_if_compatible(key, param_type, default) + except ValueError as e: + raise ValueError("%s. %s" % (fail_msg, e)) + + return getattr(self, key) + + return default + + def __contains__(self, key): + return key in self._hparam_types + + def __str__(self): + return str(sorted(self.values().items())) + + def __repr__(self): + return "%s(%s)" % (type(self).__name__, self.__str__()) + + @staticmethod + def _get_kind_name(param_type, is_list): + """Returns the field name given parameter type and is_list. + + Args: + param_type: Data type of the hparam. + is_list: Whether this is a list. + + Returns: + A string representation of the field name. + + Raises: + ValueError: If parameter type is not recognized. + """ + if issubclass(param_type, bool): + # This check must happen before issubclass(param_type, six.integer_types), + # since Python considers bool to be a subclass of int. + typename = "bool" + elif issubclass(param_type, six.integer_types): + # Setting 'int' and 'long' types to be 'int64' to ensure the type is + # compatible with both Python2 and Python3. + typename = "int64" + elif issubclass(param_type, (six.string_types, six.binary_type)): + # Setting 'string' and 'bytes' types to be 'bytes' to ensure the type is + # compatible with both Python2 and Python3. + typename = "bytes" + elif issubclass(param_type, float): + typename = "float" + else: + raise ValueError("Unsupported parameter type: %s" % str(param_type)) + + suffix = "list" if is_list else "value" + return "_".join([typename, suffix]) + + def instantiate(self): + assert "cls" in self._hparam_types + assert self.cls is not None + return self.cls(self) + + def append(self, hp): + assert type(hp) == HParams + for key, val in hp.values().items(): + if key != "cls": + self.add_hparam(key, val) + +def register_and_parse_hparams(default_config: dict, config=None, **kwargs): + """ register default config and parse""" + hparams = HParams(**kwargs) + for keys in default_config: + hparams.add_hparam(keys, default_config[keys]) + if config is not None: + hparams.override_from_dict(config) + return hparams diff --git a/athena/utils/hparam_test.py b/athena/utils/hparam_test.py new file mode 100644 index 00000000..394b5b67 --- /dev/null +++ b/athena/utils/hparam_test.py @@ -0,0 +1,521 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# Forked with minor changes from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/training/python/training/hparam_test.py pylint: disable=line-too-long +"""Tests for hparam.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from transform.utils import hparam + +import tensorflow as tf + + +class HParamsTest(tf.test.TestCase): + def testEmpty(self): + hparams = hparam.HParams() + self.assertDictEqual({}, hparams.values()) + hparams.parse("") + self.assertDictEqual({}, hparams.values()) + with self.assertRaisesRegexp(ValueError, "Unknown hyperparameter"): + hparams.parse("xyz=123") + + def testContains(self): + hparams = hparam.HParams(foo=1) + self.assertTrue("foo" in hparams) + self.assertFalse("bar" in hparams) + + def testSomeValues(self): + hparams = hparam.HParams(aaa=1, b=2.0, c_c="relu6", d="/a/b=c/d") + self.assertDictEqual( + {"aaa": 1, "b": 2.0, "c_c": "relu6", "d": "/a/b=c/d"}, hparams.values() + ) + expected_str = ( + "[('aaa', 1), ('b', 2.0), ('c_c', 'relu6'), " "('d', '/a/b=c/d')]" + ) + self.assertEqual(expected_str, str(hparams.__str__())) + self.assertEqual(expected_str, str(hparams)) + self.assertEqual(1, hparams.aaa) + self.assertEqual(2.0, hparams.b) + self.assertEqual("relu6", hparams.c_c) + self.assertEqual("/a/b=c/d", hparams.d) + hparams.parse("aaa=12") + self.assertDictEqual( + {"aaa": 12, "b": 2.0, "c_c": "relu6", "d": "/a/b=c/d"}, hparams.values() + ) + self.assertEqual(12, hparams.aaa) + self.assertEqual(2.0, hparams.b) + self.assertEqual("relu6", hparams.c_c) + self.assertEqual("/a/b=c/d", hparams.d) + hparams.parse("c_c=relu4, b=-2.0e10") + self.assertDictEqual( + {"aaa": 12, "b": -2.0e10, "c_c": "relu4", "d": "/a/b=c/d"}, hparams.values() + ) + self.assertEqual(12, hparams.aaa) + self.assertEqual(-2.0e10, hparams.b) + self.assertEqual("relu4", hparams.c_c) + self.assertEqual("/a/b=c/d", hparams.d) + hparams.parse("c_c=,b=0,") + self.assertDictEqual( + {"aaa": 12, "b": 0, "c_c": "", "d": "/a/b=c/d"}, hparams.values() + ) + self.assertEqual(12, hparams.aaa) + self.assertEqual(0.0, hparams.b) + self.assertEqual("", hparams.c_c) + self.assertEqual("/a/b=c/d", hparams.d) + hparams.parse('c_c=2.3",b=+2,') + self.assertEqual(2.0, hparams.b) + self.assertEqual('2.3"', hparams.c_c) + hparams.parse("d=/a/b/c/d,aaa=11,") + self.assertEqual(11, hparams.aaa) + self.assertEqual(2.0, hparams.b) + self.assertEqual('2.3"', hparams.c_c) + self.assertEqual("/a/b/c/d", hparams.d) + hparams.parse("b=1.5,d=/a=b/c/d,aaa=10,") + self.assertEqual(10, hparams.aaa) + self.assertEqual(1.5, hparams.b) + self.assertEqual('2.3"', hparams.c_c) + self.assertEqual("/a=b/c/d", hparams.d) + with self.assertRaisesRegexp(ValueError, "Unknown hyperparameter"): + hparams.parse("x=123") + with self.assertRaisesRegexp(ValueError, "Could not parse"): + hparams.parse("aaa=poipoi") + with self.assertRaisesRegexp(ValueError, "Could not parse"): + hparams.parse("aaa=1.0") + with self.assertRaisesRegexp(ValueError, "Could not parse"): + hparams.parse("b=12x") + with self.assertRaisesRegexp(ValueError, "Could not parse"): + hparams.parse("b=relu") + with self.assertRaisesRegexp(ValueError, "Must not pass a list"): + hparams.parse("aaa=[123]") + self.assertEqual(10, hparams.aaa) + self.assertEqual(1.5, hparams.b) + self.assertEqual('2.3"', hparams.c_c) + self.assertEqual("/a=b/c/d", hparams.d) + + def testWithPeriodInVariableName(self): + hparams = hparam.HParams() + hparams.add_hparam(name="a.b", value=0.0) + hparams.parse("a.b=1.0") + self.assertEqual(1.0, getattr(hparams, "a.b")) + hparams.add_hparam(name="c.d", value=0.0) + with self.assertRaisesRegexp(ValueError, "Could not parse"): + hparams.parse("c.d=abc") + hparams.add_hparam(name="e.f", value="") + hparams.parse("e.f=abc") + self.assertEqual("abc", getattr(hparams, "e.f")) + hparams.add_hparam(name="d..", value=0.0) + hparams.parse("d..=10.0") + self.assertEqual(10.0, getattr(hparams, "d..")) + + def testSetFromMap(self): + hparams = hparam.HParams(a=1, b=2.0, c="tanh") + hparams.override_from_dict({"a": -2, "c": "identity"}) + self.assertDictEqual({"a": -2, "c": "identity", "b": 2.0}, hparams.values()) + + hparams = hparam.HParams(x=1, b=2.0, d=[0.5]) + hparams.override_from_dict({"d": [0.1, 0.2, 0.3]}) + self.assertDictEqual({"d": [0.1, 0.2, 0.3], "x": 1, "b": 2.0}, hparams.values()) + + def testFunction(self): + def f(x): + return x + + hparams = hparam.HParams(function=f) + self.assertEqual(hparams.function, f) + + json_str = hparams.to_json() + self.assertEqual(json_str, "{}") + + def testBoolParsing(self): + for value in "true", "false", "True", "False", "1", "0": + for initial in False, True: + hparams = hparam.HParams(use_gpu=initial) + hparams.parse("use_gpu=" + value) + self.assertEqual(hparams.use_gpu, value in ["True", "true", "1"]) + + def testBoolParsingFail(self): + hparams = hparam.HParams(use_gpu=True) + with self.assertRaisesRegexp(ValueError, r"Could not parse.*use_gpu"): + hparams.parse("use_gpu=yep") + + def testLists(self): + hparams = hparam.HParams(aaa=[1], b=[2.0, 3.0], c_c=["relu6"]) + self.assertDictEqual( + {"aaa": [1], "b": [2.0, 3.0], "c_c": ["relu6"]}, hparams.values() + ) + self.assertEqual([1], hparams.aaa) + self.assertEqual([2.0, 3.0], hparams.b) + self.assertEqual(["relu6"], hparams.c_c) + hparams.parse("aaa=[12]") + self.assertEqual([12], hparams.aaa) + hparams.parse("aaa=[12,34,56]") + self.assertEqual([12, 34, 56], hparams.aaa) + hparams.parse("c_c=[relu4,relu12],b=[1.0]") + self.assertEqual(["relu4", "relu12"], hparams.c_c) + self.assertEqual([1.0], hparams.b) + hparams.parse("c_c=[],aaa=[-34]") + self.assertEqual([-34], hparams.aaa) + self.assertEqual([], hparams.c_c) + hparams.parse("c_c=[_12,3'4\"],aaa=[+3]") + self.assertEqual([3], hparams.aaa) + self.assertEqual(["_12", "3'4\""], hparams.c_c) + with self.assertRaisesRegexp(ValueError, "Unknown hyperparameter"): + hparams.parse("x=[123]") + with self.assertRaisesRegexp(ValueError, "Could not parse"): + hparams.parse("aaa=[poipoi]") + with self.assertRaisesRegexp(ValueError, "Could not parse"): + hparams.parse("aaa=[1.0]") + with self.assertRaisesRegexp(ValueError, "Could not parse"): + hparams.parse("b=[12x]") + with self.assertRaisesRegexp(ValueError, "Could not parse"): + hparams.parse("b=[relu]") + with self.assertRaisesRegexp(ValueError, "Must pass a list"): + hparams.parse("aaa=123") + + def testParseValuesWithIndexAssigment1(self): + """Assignment to an index position.""" + parse_dict = hparam.parse_values("arr[1]=10", {"arr": int}) + self.assertEqual(len(parse_dict), 1) + self.assertIsInstance(parse_dict["arr"], dict) + self.assertDictEqual(parse_dict["arr"], {1: 10}) + + def testParseValuesWithIndexAssigment1_IgnoreUnknown(self): + """Assignment to an index position.""" + parse_dict = hparam.parse_values( + "arr[1]=10,b=5", {"arr": int}, ignore_unknown=True + ) + self.assertEqual(len(parse_dict), 1) + self.assertIsInstance(parse_dict["arr"], dict) + self.assertDictEqual(parse_dict["arr"], {1: 10}) + + def testParseValuesWithIndexAssigment2(self): + """Assignment to multiple index positions.""" + parse_dict = hparam.parse_values("arr[0]=10,arr[5]=20", {"arr": int}) + self.assertEqual(len(parse_dict), 1) + self.assertIsInstance(parse_dict["arr"], dict) + self.assertDictEqual(parse_dict["arr"], {0: 10, 5: 20}) + + def testParseValuesWithIndexAssigment2_IgnoreUnknown(self): + """Assignment to multiple index positions.""" + parse_dict = hparam.parse_values( + "arr[0]=10,arr[5]=20,foo=bar", {"arr": int}, ignore_unknown=True + ) + self.assertEqual(len(parse_dict), 1) + self.assertIsInstance(parse_dict["arr"], dict) + self.assertDictEqual(parse_dict["arr"], {0: 10, 5: 20}) + + def testParseValuesWithIndexAssigment3(self): + """Assignment to index positions in multiple names.""" + parse_dict = hparam.parse_values( + "arr[0]=10,arr[1]=20,L[5]=100,L[10]=200", {"arr": int, "L": int} + ) + self.assertEqual(len(parse_dict), 2) + self.assertIsInstance(parse_dict["arr"], dict) + self.assertDictEqual(parse_dict["arr"], {0: 10, 1: 20}) + self.assertIsInstance(parse_dict["L"], dict) + self.assertDictEqual(parse_dict["L"], {5: 100, 10: 200}) + + def testParseValuesWithIndexAssigment3_IgnoreUnknown(self): + """Assignment to index positions in multiple names.""" + parse_dict = hparam.parse_values( + "arr[0]=10,C=5,arr[1]=20,B[0]=kkk,L[5]=100,L[10]=200", + {"arr": int, "L": int}, + ignore_unknown=True, + ) + self.assertEqual(len(parse_dict), 2) + self.assertIsInstance(parse_dict["arr"], dict) + self.assertDictEqual(parse_dict["arr"], {0: 10, 1: 20}) + self.assertIsInstance(parse_dict["L"], dict) + self.assertDictEqual(parse_dict["L"], {5: 100, 10: 200}) + + def testParseValuesWithIndexAssigment4(self): + """Assignment of index positions and scalars.""" + parse_dict = hparam.parse_values( + "x=10,arr[1]=20,y=30", {"x": int, "y": int, "arr": int} + ) + self.assertEqual(len(parse_dict), 3) + self.assertIsInstance(parse_dict["arr"], dict) + self.assertDictEqual(parse_dict["arr"], {1: 20}) + self.assertEqual(parse_dict["x"], 10) + self.assertEqual(parse_dict["y"], 30) + + def testParseValuesWithIndexAssigment4_IgnoreUnknown(self): + """Assignment of index positions and scalars.""" + parse_dict = hparam.parse_values( + "x=10,foo[0]=bar,arr[1]=20,zzz=78,y=30", + {"x": int, "y": int, "arr": int}, + ignore_unknown=True, + ) + self.assertEqual(len(parse_dict), 3) + self.assertIsInstance(parse_dict["arr"], dict) + self.assertDictEqual(parse_dict["arr"], {1: 20}) + self.assertEqual(parse_dict["x"], 10) + self.assertEqual(parse_dict["y"], 30) + + def testParseValuesWithIndexAssigment5(self): + """Different variable types.""" + parse_dict = hparam.parse_values( + "a[0]=5,b[1]=true,c[2]=abc,d[3]=3.14", + {"a": int, "b": bool, "c": str, "d": float}, + ) + self.assertEqual(set(parse_dict.keys()), {"a", "b", "c", "d"}) + self.assertIsInstance(parse_dict["a"], dict) + self.assertDictEqual(parse_dict["a"], {0: 5}) + self.assertIsInstance(parse_dict["b"], dict) + self.assertDictEqual(parse_dict["b"], {1: True}) + self.assertIsInstance(parse_dict["c"], dict) + self.assertDictEqual(parse_dict["c"], {2: "abc"}) + self.assertIsInstance(parse_dict["d"], dict) + self.assertDictEqual(parse_dict["d"], {3: 3.14}) + + def testParseValuesWithIndexAssigment5_IgnoreUnknown(self): + """Different variable types.""" + parse_dict = hparam.parse_values( + "a[0]=5,cc=4,b[1]=true,c[2]=abc,mm=2,d[3]=3.14", + {"a": int, "b": bool, "c": str, "d": float}, + ignore_unknown=True, + ) + self.assertEqual(set(parse_dict.keys()), {"a", "b", "c", "d"}) + self.assertIsInstance(parse_dict["a"], dict) + self.assertDictEqual(parse_dict["a"], {0: 5}) + self.assertIsInstance(parse_dict["b"], dict) + self.assertDictEqual(parse_dict["b"], {1: True}) + self.assertIsInstance(parse_dict["c"], dict) + self.assertDictEqual(parse_dict["c"], {2: "abc"}) + self.assertIsInstance(parse_dict["d"], dict) + self.assertDictEqual(parse_dict["d"], {3: 3.14}) + + def testParseValuesWithBadIndexAssigment1(self): + """Reject assignment of list to variable type.""" + with self.assertRaisesRegexp( + ValueError, r"Assignment of a list to a list index." + ): + hparam.parse_values("arr[1]=[1,2,3]", {"arr": int}) + + def testParseValuesWithBadIndexAssigment1_IgnoreUnknown(self): + """Reject assignment of list to variable type.""" + with self.assertRaisesRegexp( + ValueError, r"Assignment of a list to a list index." + ): + hparam.parse_values("arr[1]=[1,2,3],c=8", {"arr": int}, ignore_unknown=True) + + def testParseValuesWithBadIndexAssigment2(self): + """Reject if type missing.""" + with self.assertRaisesRegexp( + ValueError, r"Unknown hyperparameter type for arr" + ): + hparam.parse_values("arr[1]=5", {}) + + def testParseValuesWithBadIndexAssigment2_IgnoreUnknown(self): + """Ignore missing type.""" + hparam.parse_values("arr[1]=5", {}, ignore_unknown=True) + + def testParseValuesWithBadIndexAssigment3(self): + """Reject type of the form name[index].""" + with self.assertRaisesRegexp(ValueError, "Unknown hyperparameter type for arr"): + hparam.parse_values("arr[1]=1", {"arr[1]": int}) + + def testParseValuesWithBadIndexAssigment3_IgnoreUnknown(self): + """Ignore type of the form name[index].""" + hparam.parse_values("arr[1]=1", {"arr[1]": int}, ignore_unknown=True) + + def testWithReusedVariables(self): + with self.assertRaisesRegexp( + ValueError, "Multiple assignments to variable 'x'" + ): + hparam.parse_values("x=1,x=1", {"x": int}) + + with self.assertRaisesRegexp( + ValueError, "Multiple assignments to variable 'arr'" + ): + hparam.parse_values("arr=[100,200],arr[0]=10", {"arr": int}) + + with self.assertRaisesRegexp( + ValueError, r"Multiple assignments to variable \'arr\[0\]\'" + ): + hparam.parse_values("arr[0]=10,arr[0]=20", {"arr": int}) + + with self.assertRaisesRegexp( + ValueError, "Multiple assignments to variable 'arr'" + ): + hparam.parse_values("arr[0]=10,arr=[100]", {"arr": int}) + + def testJson(self): + hparams = hparam.HParams(aaa=1, b=2.0, c_c="relu6", d=True) + self.assertDictEqual( + {"aaa": 1, "b": 2.0, "c_c": "relu6", "d": True}, hparams.values() + ) + self.assertEqual(1, hparams.aaa) + self.assertEqual(2.0, hparams.b) + self.assertEqual("relu6", hparams.c_c) + hparams.parse_json('{"aaa": 12, "b": 3.0, "c_c": "relu4", "d": false}') + self.assertDictEqual( + {"aaa": 12, "b": 3.0, "c_c": "relu4", "d": False}, hparams.values() + ) + self.assertEqual(12, hparams.aaa) + self.assertEqual(3.0, hparams.b) + self.assertEqual("relu4", hparams.c_c) + + json_str = hparams.to_json() + hparams2 = hparam.HParams(aaa=10, b=20.0, c_c="hello", d=False) + hparams2.parse_json(json_str) + self.assertEqual(12, hparams2.aaa) + self.assertEqual(3.0, hparams2.b) + self.assertEqual("relu4", hparams2.c_c) + self.assertEqual(False, hparams2.d) + + hparams3 = hparam.HParams(aaa=123) + self.assertEqual('{"aaa": 123}', hparams3.to_json()) + self.assertEqual('{\n "aaa": 123\n}', hparams3.to_json(indent=2)) + self.assertEqual('{"aaa"=123}', hparams3.to_json(separators=(";", "="))) + + hparams4 = hparam.HParams(aaa=123, b="hello", c_c=False) + self.assertEqual( + '{"aaa": 123, "b": "hello", "c_c": false}', hparams4.to_json(sort_keys=True) + ) + + def testSetHParam(self): + hparams = hparam.HParams(aaa=1, b=2.0, c_c="relu6", d=True) + self.assertDictEqual( + {"aaa": 1, "b": 2.0, "c_c": "relu6", "d": True}, hparams.values() + ) + self.assertEqual(1, hparams.aaa) + self.assertEqual(2.0, hparams.b) + self.assertEqual("relu6", hparams.c_c) + + hparams.set_hparam("aaa", 12) + hparams.set_hparam("b", 3.0) + hparams.set_hparam("c_c", "relu4") + hparams.set_hparam("d", False) + self.assertDictEqual( + {"aaa": 12, "b": 3.0, "c_c": "relu4", "d": False}, hparams.values() + ) + self.assertEqual(12, hparams.aaa) + self.assertEqual(3.0, hparams.b) + self.assertEqual("relu4", hparams.c_c) + + def testSetHParamListNonListMismatch(self): + hparams = hparam.HParams(a=1, b=[2.0, 3.0]) + with self.assertRaisesRegexp(ValueError, r"Must not pass a list"): + hparams.set_hparam("a", [1.0]) + with self.assertRaisesRegexp(ValueError, r"Must pass a list"): + hparams.set_hparam("b", 1.0) + + def testSetHParamTypeMismatch(self): + hparams = hparam.HParams( + int_=1, str_="str", bool_=True, float_=1.1, list_int=[1, 2], none=None + ) + + with self.assertRaises(ValueError): + hparams.set_hparam("str_", 2.2) + + with self.assertRaises(ValueError): + hparams.set_hparam("int_", False) + + with self.assertRaises(ValueError): + hparams.set_hparam("bool_", 1) + + with self.assertRaises(ValueError): + hparams.set_hparam("int_", 2.2) + + with self.assertRaises(ValueError): + hparams.set_hparam("list_int", [2, 3.3]) + + with self.assertRaises(ValueError): + hparams.set_hparam("int_", "2") + + # Casting int to float is OK + hparams.set_hparam("float_", 1) + + # Getting stuck with NoneType :( + hparams.set_hparam("none", "1") + self.assertEqual("1", hparams.none) + + def testGet(self): + hparams = hparam.HParams(aaa=1, b=2.0, c_c="relu6", d=True, e=[5.0, 6.0]) + + # Existing parameters with default=None. + self.assertEqual(1, hparams.get("aaa")) + self.assertEqual(2.0, hparams.get("b")) + self.assertEqual("relu6", hparams.get("c_c")) + self.assertEqual(True, hparams.get("d")) + self.assertEqual([5.0, 6.0], hparams.get("e", None)) + + # Existing parameters with compatible defaults. + self.assertEqual(1, hparams.get("aaa", 2)) + self.assertEqual(2.0, hparams.get("b", 3.0)) + self.assertEqual(2.0, hparams.get("b", 3)) + self.assertEqual("relu6", hparams.get("c_c", "default")) + self.assertEqual(True, hparams.get("d", True)) + self.assertEqual([5.0, 6.0], hparams.get("e", [1.0, 2.0, 3.0])) + self.assertEqual([5.0, 6.0], hparams.get("e", [1, 2, 3])) + + # Existing parameters with incompatible defaults. + with self.assertRaises(ValueError): + hparams.get("aaa", 2.0) + + with self.assertRaises(ValueError): + hparams.get("b", False) + + with self.assertRaises(ValueError): + hparams.get("c_c", [1, 2, 3]) + + with self.assertRaises(ValueError): + hparams.get("d", "relu") + + with self.assertRaises(ValueError): + hparams.get("e", 123.0) + + with self.assertRaises(ValueError): + hparams.get("e", ["a", "b", "c"]) + + # Nonexistent parameters. + self.assertEqual(None, hparams.get("unknown")) + self.assertEqual(123, hparams.get("unknown", 123)) + self.assertEqual([1, 2, 3], hparams.get("unknown", [1, 2, 3])) + + def testDel(self): + hparams = hparam.HParams(aaa=1, b=2.0) + + with self.assertRaises(ValueError): + hparams.set_hparam("aaa", "will fail") + + with self.assertRaises(ValueError): + hparams.add_hparam("aaa", "will fail") + + hparams.del_hparam("aaa") + hparams.add_hparam("aaa", "will work") + self.assertEqual("will work", hparams.get("aaa")) + + hparams.set_hparam("aaa", "still works") + self.assertEqual("still works", hparams.get("aaa")) + + def test_parse_dict(self): + hparams = hparam.HParams(aaa=1) + hparams.parse({"b": 2.0, "c_c": "relu6", "d": True, "e": [5.0, 6.0]}, True) + + def test_append(self): + hparams = hparam.HParams(aaa=1) + hparams2 = hparam.HParams(b=2.0, c_c="relu6", d=True, e=[5.0, 6.0]) + hparams.append(hparams2) + print(hparams) + + +if __name__ == "__main__": + tf.test.main() diff --git a/athena/utils/learning_rate.py b/athena/utils/learning_rate.py new file mode 100644 index 00000000..e6ff3356 --- /dev/null +++ b/athena/utils/learning_rate.py @@ -0,0 +1,131 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Only support eager mode +# pylint: disable=too-few-public-methods, no-member, too-many-arguments, unused-argument +""" learning rate """ +import tensorflow as tf +from ..utils.hparam import register_and_parse_hparams + + +class WarmUpLearningSchedule(tf.keras.optimizers.schedules.LearningRateSchedule): + """ WarmUp Learning rate schedule for Adam + + Used as : + optimizer = tf.keras.optimizers.Adam(learning_rate = WarmUpLearningSchedule(512), + beta_1=0.9, beta_2=0.98, epsilon=1e-9) + Args : + model_dim is the something related to total model parameters + warmup_steps is the highest learning rate iters + Returns: + return the learning rate + Idea from the paper: Attention Is All You Need + """ + + def __init__(self, model_dim=512, warmup_steps=4000, k=1.0, + decay_steps=99999999, decay_rate=1.0): + super().__init__() + + self.model_dim = tf.cast(model_dim, tf.float32) + self.warmup_steps = warmup_steps + self.k = k + self.decay_steps = tf.cast(decay_steps, tf.float32) + self.decay_rate = tf.cast(decay_rate, tf.float32) + + def __call__(self, step): + step = tf.cast(step, tf.float32) + arg1 = tf.math.rsqrt(step) + arg2 = step * (self.warmup_steps ** -1.5) + k = self.k * tf.cast(self.decay_rate ** (step // self.decay_steps), tf.float32) + + return k * tf.math.rsqrt(self.model_dim) * tf.math.minimum(arg1, arg2) + + +class WarmUpAdam(tf.keras.optimizers.Adam): + """WarmUpAdam Implementation """ + default_config = { + "d_model": 512, + "warmup_steps": 8000, + "k": 0.5, + "decay_steps": 100000, + "decay_rate": 1.0 + } + def __init__(self, config=None, beta_1=0.9, beta_2=0.999, epsilon=1e-7, + amsgrad=False, name="WarmUpAdam", **kwargs): + self.hparams = register_and_parse_hparams(self.default_config, config, cls=self.__class__) + super().__init__( + learning_rate=WarmUpLearningSchedule( + self.hparams.d_model, + self.hparams.warmup_steps, + self.hparams.k, + self.hparams.decay_steps, + self.hparams.decay_rate + ), + beta_1=beta_1, + beta_2=beta_2, + epsilon=epsilon, + amsgrad=amsgrad, + name=name, + ) + + +class ExponentialDecayLearningRateSchedule(tf.keras.optimizers.schedules.LearningRateSchedule): + """ ExponentialDecayLearningRateSchedule + + Used as : + optimizer = tf.keras.optimizers.Adam( + learning_rate = ExponentialDecayLearningRate(0.01, 100)) + Args : + initial_lr, decay_steps + Returns: + initial_lr * (0.5 ** (step // decay_steps)) + """ + + def __init__(self, initial_lr=0.005, decay_steps=10000, decay_rate=0.5): + super().__init__() + self.initial_lr = initial_lr + self.decay_steps = tf.cast(decay_steps, tf.float32) + self.decay_rate = tf.cast(decay_rate, tf.float32) + + def __call__(self, step): + step = tf.cast(step, tf.float32) + factor = tf.cast(self.decay_rate ** (step // self.decay_steps), tf.float32) + return self.initial_lr * factor + + +class ExponentialDecayAdam(tf.keras.optimizers.Adam): + """WarmUpAdam Implementation """ + default_config = { + "initial_lr": 0.005, + "decay_steps": 10000, + "decay_rate": 0.5 + } + + def __init__(self, config=None, beta_1=0.9, beta_2=0.999, epsilon=1e-7, + amsgrad=False, name="WarmUpAdam", **kwargs): + self.hparams = register_and_parse_hparams(self.default_config, config, cls=self.__class__) + super().__init__( + learning_rate=ExponentialDecayLearningRateSchedule( + self.hparams.initial_lr, + self.hparams.decay_steps, + self.hparams.decay_rate + ), + beta_1=beta_1, + beta_2=beta_2, + epsilon=epsilon, + amsgrad=amsgrad, + name=name, + ) diff --git a/athena/utils/metric_check.py b/athena/utils/metric_check.py new file mode 100644 index 00000000..c68a63a5 --- /dev/null +++ b/athena/utils/metric_check.py @@ -0,0 +1,104 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""MetricChecker""" +import time +import tensorflow as tf +import numpy as np + + +class MetricChecker: + """Hold and save best metric checkpoint + Args: + name: MetricChecker name + maximum: more greater more better + """ + + def __init__(self, optimizer): + self.best_loss = tf.constant(np.inf) + self.optimizer = optimizer + self.time_last_call = time.time() + self.steps_last_call = 0 + + def __call__(self, loss, metrics, evaluate_epoch=-1): + """summary the basic metrics like loss, lr + Args: + loss: + matrics: average loss of all previous steps in one epoch + if training is False, it must be provided + evaluate_epoch: + if evaluate_epoch >= 0: + if evaluate_epoch == -1: + if evaluate_epoch < -1: (no tf.summary.write) + Returns: + logging_str: return average and best(if improved) loss if training is False + """ + if evaluate_epoch is -1: + return self.summary_train(loss, metrics) + return self.summary_evaluate(loss, metrics, evaluate_epoch) + + def summary_train(self, loss, metrics): + """ generate summary of learning_rate, loss, metrics, speed and write on Tensorboard + """ + global_steps = tf.convert_to_tensor(self.optimizer.iterations) + learning_rate = ( + self.optimizer.lr + if isinstance(self.optimizer.lr, tf.Variable) + else (self.optimizer.lr(global_steps)) + ) + + tf.summary.scalar("loss", loss, step=global_steps) + tf.summary.scalar("learning_rate", learning_rate, step=global_steps) + for name in metrics: + metric = metrics[name] + tf.summary.scalar(name, metric, step=global_steps) + + reports = "" + reports += "global_steps: %d\t" % (global_steps) + reports += "learning_rate: %.4e\t" % (learning_rate) + reports += "loss: %.4f\t" % (loss) + for name in metrics: + metric = metrics[name] + reports += "%s: %.4f\t" % (name, metric) + right_now = time.time() + duration = right_now - self.time_last_call + self.time_last_call = right_now + if self.steps_last_call != 0: + # average duration over log_interval steps + sec_per_iter = duration / tf.cast( + (global_steps - self.steps_last_call), tf.float32 + ) + reports += "sec/iter: %.4f" % (sec_per_iter) + self.steps_last_call = global_steps + + return reports + + def summary_evaluate(self, loss, metrics, epoch=-1): + """ If epoch > 0, return a summary of loss and metrics on dev set and write on Tensorboard + Otherwise, just return evaluate loss and metrics + """ + reports = "" + if epoch >= 0: + tf.summary.scalar("evaluate_loss", loss, step=epoch) + for name in metrics: + metric = metrics[name] + tf.summary.scalar("evaluate_" + name, metric, step=epoch) + reports += "epoch: %d\t" % (epoch) + reports += "loss: %.4f\t" % (loss) + for name in metrics: + metric = metrics[name] + reports += "%s: %.4f\t" % (name, metric) + return reports diff --git a/athena/utils/misc.py b/athena/utils/misc.py new file mode 100644 index 00000000..e88db4a8 --- /dev/null +++ b/athena/utils/misc.py @@ -0,0 +1,167 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=missing-function-docstring, invalid-name +""" misc """ +import os +import wave +import tensorflow as tf +from absl import logging +import numpy as np + + +def mask_index_from_labels(labels, index): + mask = tf.math.logical_not(tf.math.equal(labels, index)) + mask = tf.cast(mask, dtype=labels.dtype) + return labels * mask + + +def insert_sos_in_labels(labels, sos): + sos = tf.ones([tf.shape(labels)[0], 1], dtype=labels.dtype) * sos + return tf.concat([sos, labels], axis=-1) + + +def remove_eos_in_labels(input_labels, labels_length): + """ remove eos in labels, batch size should be larger than 1 + assuming 0 as the padding and the last one is the eos + """ + labels = input_labels[:, :-1] + length = labels_length - 1 + max_length = tf.shape(labels)[1] + mask = tf.sequence_mask(length, max_length, dtype=labels.dtype) + labels = labels * mask + labels.set_shape([None, None]) + return labels + + +def insert_eos_in_labels(input_labels, eos, labels_length): + """ insert eos in labels, batch size should be larger than 1 + assuming 0 as the padding, + """ + zero = tf.zeros([tf.shape(input_labels)[0], 1], dtype=input_labels.dtype) + labels = tf.concat([input_labels, zero], axis=-1) + labels += tf.one_hot(labels_length, tf.shape(labels)[1], dtype=labels.dtype) * eos + return labels + + +def generate_square_subsequent_mask(size): + """ Generate a square mask for the sequence. The masked positions are filled with float(1.0). + Unmasked positions are filled with float(0.0). + """ + mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0) + return mask + + +def validate_seqs(seqs, eos): + """ Discard end symbol and elements after end symbol + Args: + seqs: tf.Tensor shape=(batch_size, seq_length) + Returns: + validated_preds: tf.SparseTensor + """ + eos = tf.cast(eos, tf.int64) + if eos != 0: + indexes = tf.TensorArray(tf.bool, size=0, dynamic_size=True) + a = tf.not_equal(eos, seqs) + res = a[:, 0] + indexes = indexes.write(0, res) + for i in tf.range(1, tf.shape(a)[1]): + res = tf.logical_and(res, a[:, i]) + indexes = indexes.write(i, res) + res = tf.transpose(indexes.stack(), [1, 0]) + validated_preds = tf.where(tf.logical_not(res), tf.zeros_like(seqs), seqs) + validated_preds = tf.sparse.from_dense(validated_preds) + else: + validated_preds = tf.where(tf.equal(eos, seqs), tf.zeros_like(seqs), seqs) + validated_preds = tf.sparse.from_dense(validated_preds) + counter = tf.cast(tf.shape(validated_preds.values)[0], tf.float32) + return validated_preds, counter + + +def get_wave_file_length(wave_file): + """ get the wave file length(duration) in ms + + :param wave_file: the path of wave file + :return: the length(ms) of the wave file + """ + if not os.path.exists(wave_file): + logging.warning("Wave file {} does not exist!".format(wave_file)) + return 0 + with wave.open(wave_file) as wav_file: + wav_frames = wav_file.getnframes() + wav_frame_rate = wav_file.getframerate() + wav_length = int(wav_frames / wav_frame_rate * 1000) # get wave duration in ms + return wav_length + + +def splice_numpy(x, context): + """ + Splice a tensor along the last dimension with context. + e.g.: + t = [[[1, 2, 3], + [4, 5, 6], + [7, 8, 9]]] + splice_tensor(t, [0, 1]) = + [[[1, 2, 3, 4, 5, 6], + [4, 5, 6, 7, 8, 9], + [7, 8, 9, 7, 8, 9]]] + + Args: + tensor: a tf.Tensor with shape (B, T, D) a.k.a. (N, H, W) + context: a list of context offsets + + Returns: + spliced tensor with shape (..., D * len(context)) + """ + # numpy can speed up 10% + x = x.numpy() + input_shape = np.shape(x) + B, T = input_shape[0], input_shape[1] + context_len = len(context) + left_boundary = -1 * min(context) if min(context) < 0 else 0 + right_boundary = max(context) if max(context) > 0 else 0 + sample_range = ([0] * left_boundary + [i for i in range(T)] + [T - 1] * right_boundary) + array = [] + for idx in range(context_len): + pos = context[idx] + if pos < 0: + pos = len(sample_range) - T - max(context) + pos + sliced = x[:, sample_range[pos : pos + T], :] + array += [sliced] + spliced = np.concatenate([i[:, :, np.newaxis, :] for i in array], axis=2) + spliced = np.reshape(spliced, (B, T, -1)) + return tf.convert_to_tensor(spliced) + +def set_default_summary_writer(summary_directory=None): + if summary_directory is None: + summary_directory = os.path.join(os.path.expanduser("~"), ".athena") + summary_directory = os.path.join(summary_directory, "event") + writer = tf.summary.create_file_writer(summary_directory) + writer.set_as_default() + +def tensor_shape(tensor): + """ Return a list with tensor shape. For each dimension, + use tensor.get_shape() first. If not available, use tf.shape(). + """ + if tensor.get_shape().dims is None: + return tf.shape(tensor) + shape_value = tensor.get_shape().as_list() + shape_tensor = tf.shape(tensor) + ret = [shape_tensor[idx] + if shape_value[idx] is None + else shape_value[idx] + for idx in range(len(shape_value))] + return ret diff --git a/athena/utils/vocabs/ch-en.vocab b/athena/utils/vocabs/ch-en.vocab new file mode 100644 index 00000000..77e89163 --- /dev/null +++ b/athena/utils/vocabs/ch-en.vocab @@ -0,0 +1,5005 @@ + 0 +么 1 +什 2 +一 3 +是 4 +小 5 +大 6 +怎 7 +有 8 +人 9 +片 10 +天 11 +二 12 +我 13 +电 14 +子 15 +三 16 +图 17 +语 18 +女 19 +词 20 +下 21 +视 22 +年 23 +之 24 +生 25 +游 26 +十 27 +手 28 +文 29 +五 30 +中 31 +多 32 +英 33 +不 34 +四 35 +戏 36 +国 37 +哪 38 +字 39 +吗 40 +动 41 +机 42 +上 43 +个 44 +学 45 +美 46 +车 47 +少 48 +好 49 +影 50 +第 51 +画 52 +歌 53 +频 54 +零 55 +新 56 +成 57 +意 58 +在 59 +儿 60 +爱 61 +看 62 +宝 63 +载 64 +用 65 +版 66 +作 67 +思 68 +义 69 +全 70 +战 71 +公 72 +说 73 +花 74 +头 75 +最 76 +六 77 +世 78 +和 79 +写 80 +百 81 +神 82 +要 83 +主 84 +集 85 +王 86 +里 87 +能 88 +于 89 +样 90 +级 91 +地 92 +了 93 +做 94 +特 95 +出 96 +山 97 +气 98 +物 99 +春 100 +界 101 +家 102 +以 103 +可 104 +日 105 +水 106 +乐 107 +西 108 +关 109 +名 110 +星 111 +长 112 +事 113 +时 114 +海 115 +金 116 +色 117 +法 118 +心 119 +九 120 +奇 121 +打 122 +句 123 +八 124 +龙 125 +开 126 +风 127 +吃 128 +老 129 +七 130 +曲 131 +你 132 +到 133 +近 134 +会 135 +高 136 +情 137 +火 138 +方 139 +熊 140 +没 141 +剧 142 +后 143 +为 144 +来 145 +音 146 +羊 147 +月 148 +号 149 +记 150 +明 151 +照 152 +果 153 +爸 154 +网 155 +奥 156 +安 157 +报 158 +跑 159 +市 160 +节 161 +南 162 +发 163 +话 164 +东 165 +男 166 +组 167 +面 168 +魔 169 +黄 170 +点 171 +马 172 +梦 173 +鱼 174 +读 175 +红 176 +州 177 +白 178 +反 179 +猪 180 +舞 181 +数 182 +谁 183 +妈 184 +传 185 +如 186 +士 187 +城 188 +分 189 +侠 190 +些 191 +曼 192 +拉 193 +千 194 +快 195 +解 196 +过 197 +几 198 +阳 199 +光 200 +飞 201 +行 202 +无 203 +比 204 +单 205 +玩 206 +线 207 +想 208 +部 209 +场 210 +喜 211 +自 212 +加 213 +季 214 +巴 215 +钱 216 +化 217 +诗 218 +书 219 +感 220 +尸 221 +仙 222 +性 223 +本 224 +体 225 +与 226 +北 227 +信 228 +孩 229 +江 230 +广 231 +酷 232 +雪 233 +演 234 +古 235 +去 236 +平 237 +吧 238 +甲 239 +赛 240 +僵 241 +道 242 +间 243 +形 244 +度 245 +表 246 +故 247 +期 248 +米 249 +对 250 +园 251 +超 252 +变 253 +格 254 +活 255 +啦 256 +李 257 +张 258 +器 259 +河 260 +工 261 +何 262 +香 263 +都 264 +林 265 +空 266 +品 267 +联 268 +丽 269 +题 270 +尔 271 +者 272 +雄 273 +路 274 +代 275 +太 276 +衣 277 +民 278 +放 279 +笔 280 +漫 281 +罗 282 +童 283 +师 284 +青 285 +册 286 +结 287 +克 288 +鸡 289 +斯 290 +华 291 +门 292 +相 293 +精 294 +冰 295 +弟 296 +预 297 +身 298 +球 299 +真 300 +笑 301 +播 302 +通 303 +猫 304 +卡 305 +简 306 +服 307 +等 308 +合 309 +业 310 +重 311 +理 312 +还 313 +京 314 +口 315 +两 316 +石 317 +装 318 +强 319 +狗 320 +回 321 +黑 322 +元 323 +起 324 +植 325 +像 326 +晚 327 +院 328 +苹 329 +万 330 +流 331 +穿 332 +清 333 +教 334 +经 335 +福 336 +包 337 +价 338 +兄 339 +雨 340 +区 341 +现 342 +办 343 +内 344 +微 345 +原 346 +武 347 +式 348 +亚 349 +斗 350 +假 351 +码 352 +铁 353 +带 354 +声 355 +蛋 356 +前 357 +勇 358 +案 359 +言 360 +费 361 +灵 362 +这 363 +铠 364 +今 365 +保 366 +种 367 +破 368 +叫 369 +容 370 +击 371 +豆 372 +云 373 +力 374 +娃 375 +欢 376 +医 377 +正 378 +外 379 +唱 380 +同 381 +死 382 +造 383 +贝 384 +见 385 +票 386 +房 387 +友 388 +别 389 +彩 390 +得 391 +杨 392 +越 393 +答 394 +队 395 +草 396 +皮 397 +奔 398 +所 399 +台 400 +查 401 +险 402 +考 403 +陈 404 +牛 405 +交 406 +校 407 +周 408 +狂 409 +连 410 +习 411 +岁 412 +那 413 +们 414 +件 415 +亲 416 +双 417 +翻 418 +搜 419 +制 420 +科 421 +县 422 +叶 423 +座 424 +拼 425 +候 426 +湖 427 +奶 428 +眼 429 +血 430 +刘 431 +知 432 +具 433 +历 434 +观 435 +卫 436 +利 437 +密 438 +菜 439 +雷 440 +汽 441 +酒 442 +夜 443 +店 444 +员 445 +迪 446 +应 447 +银 448 +试 449 +木 450 +给 451 +听 452 +才 453 +肉 454 +免 455 +芭 456 +苏 457 +树 458 +运 459 +德 460 +艺 461 +目 462 +治 463 +指 464 +景 465 +毛 466 +抄 467 +当 468 +消 469 +股 470 +着 471 +狼 472 +纸 473 +译 474 +韩 475 +修 476 +婚 477 +药 478 +油 479 +被 480 +完 481 +站 482 +娘 483 +达 484 +省 485 +计 486 +干 487 +常 488 +食 489 +兵 490 +姐 491 +怪 492 +描 493 +程 494 +料 495 +啊 496 +先 497 +阿 498 +旁 499 +找 500 +然 501 +首 502 +汉 503 +鬼 504 +虎 505 +产 506 +易 507 +孕 508 +洋 509 +屁 510 +骨 511 +走 512 +请 513 +角 514 +兽 515 +直 516 +哥 517 +枪 518 +位 519 +索 520 +阅 521 +术 522 +极 523 +实 524 +进 525 +资 526 +总 527 +兰 528 +母 529 +他 530 +定 531 +处 532 +妇 533 +宁 534 +吉 535 +洗 536 +系 537 +除 538 +因 539 +初 540 +亮 541 +泰 542 +唐 543 +疯 544 +基 545 +剑 546 +村 547 +边 548 +排 549 +恐 550 +功 551 +志 552 +丝 553 +夫 554 +寒 555 +很 556 +速 557 +步 558 +兔 559 +赵 560 +招 561 +脑 562 +灰 563 +刚 564 +热 565 +列 566 +酸 567 +养 568 +算 569 +朋 570 +闻 571 +皇 572 +鸟 573 +军 574 +圣 575 +病 576 +司 577 +入 578 +盗 579 +妹 580 +郑 581 +属 582 +秘 583 +岛 584 +脱 585 +玉 586 +模 587 +问 588 +烧 589 +限 590 +冒 591 +沙 592 +型 593 +只 594 +蓝 595 +牌 596 +哈 597 +旋 598 +夏 599 +幻 600 +就 601 +农 602 +量 603 +博 604 +商 605 +建 606 +鼠 607 +汤 608 +深 609 +示 610 +裸 611 +短 612 +使 613 +警 614 +官 615 +从 616 +买 617 +喝 618 +转 619 +乡 620 +课 621 +妖 622 +野 623 +粉 624 +远 625 +偏 626 +立 627 +莉 628 +阴 629 +软 630 +秋 631 +录 632 +类 633 +土 634 +恶 635 +底 636 +次 637 +温 638 +糖 639 +菲 640 +调 641 +卖 642 +巨 643 +盟 644 +离 645 +源 646 +板 647 +拍 648 +素 649 +珠 650 +萝 651 +床 652 +管 653 +客 654 +萌 655 +接 656 +把 657 +复 658 +灯 659 +封 660 +布 661 +鞋 662 +朵 663 +护 664 +搞 665 +配 666 +优 667 +怀 668 +改 669 +营 670 +跳 671 +孙 672 +兴 673 +非 674 +将 675 +族 676 +局 677 +炎 678 +宫 679 +而 680 +婆 681 +桃 682 +命 683 +馆 684 +豪 685 +助 686 +向 687 +设 688 +操 689 +范 690 +恋 691 +技 692 +维 693 +章 694 +奖 695 +雅 696 +史 697 +凯 698 +杀 699 +换 700 +圆 701 +耳 702 +杰 703 +田 704 +整 705 +镇 706 +峰 707 +收 708 +摩 709 +逼 710 +减 711 +泡 712 +标 713 +己 714 +态 715 +虫 716 +贵 717 +缘 718 +育 719 +波 720 +环 721 +含 722 +楼 723 +朝 724 +饭 725 +落 726 +根 727 +智 728 +洛 729 +存 730 +争 731 +团 732 +托 733 +绝 734 +壁 735 +专 736 +条 737 +猎 738 +难 739 +称 740 +质 741 +介 742 +币 743 +告 744 +绿 745 +牙 746 +宠 747 +积 748 +象 749 +睡 750 +班 751 +呢 752 +欧 753 +佳 754 +息 755 +颜 756 +尼 757 +帮 758 +帝 759 +秀 760 +岭 761 +妻 762 +颖 763 +求 764 +瓜 765 +望 766 +呀 767 +拿 768 +妙 769 +置 770 +乘 771 +烟 772 +暖 773 +探 774 +激 775 +半 776 +折 777 +蹈 778 +梅 779 +川 780 +芦 781 +跟 782 +晓 783 +冬 784 +盒 785 +裤 786 +典 787 +乳 788 +旅 789 +盘 790 +港 791 +亡 792 +幸 793 +庆 794 +刀 795 +便 796 +麻 797 +证 798 +爆 799 +取 800 +肥 801 +钢 802 +威 803 +妃 804 +异 805 +漂 806 +凤 807 +暴 808 +麦 809 +防 810 +胸 811 +编 812 +提 813 +逃 814 +升 815 +背 816 +统 817 +茶 818 +雕 819 +庄 820 +康 821 +坐 822 +脚 823 +猜 824 +健 825 +丁 826 +济 827 +塔 828 +又 829 +需 830 +让 831 +腾 832 +鲁 833 +该 834 +吻 835 +陆 836 +吴 837 +至 838 +更 839 +纪 840 +蛇 841 +贼 842 +务 843 +讲 844 +榜 845 +卷 846 +侣 847 +钟 848 +其 849 +驾 850 +紫 851 +也 852 +佛 853 +俊 854 +共 855 +痛 856 +礼 857 +瑞 858 +政 859 +骑 860 +忍 861 +乌 862 +脸 863 +未 864 +洲 865 +充 866 +绵 867 +伤 868 +毒 869 +顺 870 +糕 871 +约 872 +霸 873 +练 874 +父 875 +续 876 +觉 877 +导 878 +邮 879 +迹 880 +社 881 +室 882 +篇 883 +攻 884 +套 885 +款 886 +再 887 +早 888 +登 889 +满 890 +堂 891 +葫 892 +寓 893 +松 894 +静 895 +值 896 +由 897 +奸 898 +谜 899 +甜 900 +状 901 +帅 902 +露 903 +效 904 +冲 905 +贴 906 +识 907 +冷 908 +街 909 +嘴 910 +偷 911 +询 912 +送 913 +桥 914 +普 915 +胡 916 +藏 917 +段 918 +幼 919 +浪 920 +临 921 +屏 922 +逆 923 +炒 924 +丹 925 +氧 926 +归 927 +祝 928 +嗯 929 +狙 930 +足 931 +蜜 932 +丰 933 +永 934 +注 935 +失 936 +富 937 +弹 938 +瓦 939 +袋 940 +宋 941 +蜡 942 +娜 943 +谢 944 +妆 945 +测 946 +念 947 +恩 948 +射 949 +守 950 +左 951 +姆 952 +航 953 +箱 954 +久 955 +末 956 +尾 957 +锋 958 +君 959 +乱 960 +啥 961 +选 962 +疼 963 +爷 964 +蒙 965 +宇 966 +卜 967 +始 968 +迦 969 +燕 970 +犬 971 +附 972 +宵 973 +迷 974 +巧 975 +份 976 +伦 977 +轻 978 +味 979 +住 980 +齐 981 +购 982 +琴 983 +疗 984 +滨 985 +邪 986 +莲 987 +压 988 +荣 989 +论 990 +症 991 +姑 992 +终 993 +补 994 +谱 995 +受 996 +昌 997 +轮 998 +佩 999 +各 1000 +居 1001 +尿 1002 +魂 1003 +掉 1004 +镜 1005 +鹿 1006 +徐 1007 +独 1008 +胜 1009 +讯 1010 +倒 1011 +绍 1012 +肚 1013 +锁 1014 +已 1015 +央 1016 +决 1017 +滑 1018 +块 1019 +害 1020 +势 1021 +群 1022 +际 1023 +泉 1024 +输 1025 +怒 1026 +趣 1027 +右 1028 +及 1029 +屋 1030 +材 1031 +敢 1032 +姓 1033 +液 1034 +厂 1035 +划 1036 +拳 1037 +创 1038 +征 1039 +沈 1040 +腿 1041 +突 1042 +晨 1043 +则 1044 +惊 1045 +竹 1046 +验 1047 +泽 1048 +毕 1049 +辣 1050 +悟 1051 +憨 1052 +歇 1053 +细 1054 +捕 1055 +凡 1056 +任 1057 +律 1058 +遇 1059 +狐 1060 +柴 1061 +吸 1062 +支 1063 +财 1064 +拟 1065 +柳 1066 +祖 1067 +暗 1068 +猴 1069 +聘 1070 +邓 1071 +略 1072 +职 1073 +每 1074 +嘉 1075 +津 1076 +滴 1077 +秦 1078 +若 1079 +玛 1080 +参 1081 +淘 1082 +庙 1083 +裙 1084 +良 1085 +珍 1086 +铃 1087 +敌 1088 +适 1089 +尽 1090 +呼 1091 +莫 1092 +插 1093 +鸭 1094 +锦 1095 +针 1096 +瓶 1097 +规 1098 +货 1099 +挂 1100 +炼 1101 +肖 1102 +依 1103 +苦 1104 +胶 1105 +忘 1106 +盛 1107 +释 1108 +宜 1109 +晶 1110 +谷 1111 +坏 1112 +引 1113 +架 1114 +错 1115 +络 1116 +裁 1117 +庭 1118 +炮 1119 +切 1120 +龟 1121 +莱 1122 +堡 1123 +戴 1124 +移 1125 +欲 1126 +刷 1127 +偶 1128 +摇 1129 +顶 1130 +评 1131 +炸 1132 +霆 1133 +艾 1134 +泥 1135 +蝶 1136 +急 1137 +鼻 1138 +递 1139 +凉 1140 +午 1141 +澡 1142 +领 1143 +致 1144 +朱 1145 +柯 1146 +权 1147 +众 1148 +餐 1149 +苍 1150 +降 1151 +婴 1152 +棋 1153 +某 1154 +船 1155 +伟 1156 +透 1157 +丸 1158 +腹 1159 +许 1160 +备 1161 +钻 1162 +察 1163 +印 1164 +盐 1165 +伊 1166 +低 1167 +持 1168 +诺 1169 +闹 1170 +户 1171 +遥 1172 +鲨 1173 +俗 1174 +戒 1175 +杜 1176 +纳 1177 +郎 1178 +浩 1179 +令 1180 +孔 1181 +况 1182 +杭 1183 +必 1184 +危 1185 +森 1186 +傻 1187 +寻 1188 +盖 1189 +膜 1190 +尊 1191 +控 1192 +睛 1193 +付 1194 +蛛 1195 +幂 1196 +艳 1197 +腰 1198 +派 1199 +刻 1200 +举 1201 +刑 1202 +括 1203 +敏 1204 +较 1205 +哆 1206 +鹅 1207 +杯 1208 +夺 1209 +蜂 1210 +祥 1211 +臭 1212 +断 1213 +玫 1214 +境 1215 +尚 1216 +梁 1217 +虹 1218 +郭 1219 +赞 1220 +胃 1221 +鲜 1222 +蜘 1223 +闪 1224 +签 1225 +展 1226 +悠 1227 +眉 1228 +饼 1229 +启 1230 +默 1231 +旗 1232 +湾 1233 +锅 1234 +聊 1235 +仁 1236 +侦 1237 +救 1238 +响 1239 +桂 1240 +萨 1241 +似 1242 +慧 1243 +肤 1244 +瑰 1245 +怖 1246 +填 1247 +棒 1248 +圳 1249 +待 1250 +扬 1251 +推 1252 +辽 1253 +愤 1254 +丘 1255 +燃 1256 +希 1257 +荷 1258 +徽 1259 +桌 1260 +摸 1261 +灭 1262 +址 1263 +冠 1264 +端 1265 +诚 1266 +池 1267 +籍 1268 +隐 1269 +围 1270 +溪 1271 +伴 1272 +核 1273 +杉 1274 +炫 1275 +卓 1276 +胎 1277 +煮 1278 +项 1279 +宾 1280 +狮 1281 +益 1282 +馀 1283 +塞 1284 +扫 1285 +序 1286 +止 1287 +潮 1288 +负 1289 +惠 1290 +途 1291 +樱 1292 +善 1293 +拜 1294 +凰 1295 +囊 1296 +伯 1297 +昆 1298 +咪 1299 +洞 1300 +赤 1301 +旧 1302 +熟 1303 +蒸 1304 +匙 1305 +往 1306 +准 1307 +欣 1308 +劳 1309 +腐 1310 +退 1311 +钥 1312 +愿 1313 +虚 1314 +虾 1315 +霉 1316 +批 1317 +蝴 1318 +烈 1319 +禁 1320 +饰 1321 +朗 1322 +增 1323 +横 1324 +申 1325 +追 1326 +按 1327 +捉 1328 +随 1329 +浙 1330 +般 1331 +钓 1332 +挖 1333 +胆 1334 +仪 1335 +租 1336 +舒 1337 +检 1338 +慢 1339 +闯 1340 +媚 1341 +肠 1342 +休 1343 +坦 1344 +账 1345 +拔 1346 +综 1347 +谭 1348 +沉 1349 +曹 1350 +夕 1351 +率 1352 +墙 1353 +隆 1354 +鹰 1355 +圈 1356 +厅 1357 +弄 1358 +叔 1359 +震 1360 +泳 1361 +贺 1362 +扎 1363 +刺 1364 +玲 1365 +剪 1366 +蔬 1367 +罪 1368 +恺 1369 +袜 1370 +舍 1371 +洁 1372 +叉 1373 +吹 1374 +概 1375 +绣 1376 +井 1377 +倍 1378 +湿 1379 +留 1380 +喻 1381 +投 1382 +浒 1383 +炖 1384 +净 1385 +耀 1386 +奏 1387 +染 1388 +怕 1389 +抗 1390 +梨 1391 +雀 1392 +掌 1393 +芳 1394 +凌 1395 +采 1396 +篮 1397 +碧 1398 +鼓 1399 +菇 1400 +爽 1401 +否 1402 +桶 1403 +谈 1404 +汇 1405 +纯 1406 +洪 1407 +孤 1408 +扮 1409 +墓 1410 +碳 1411 +寺 1412 +羽 1413 +仔 1414 +铜 1415 +销 1416 +构 1417 +废 1418 +例 1419 +散 1420 +键 1421 +氢 1422 +苗 1423 +认 1424 +岳 1425 +顾 1426 +傲 1427 +唯 1428 +饺 1429 +抱 1430 +陵 1431 +雾 1432 +抽 1433 +斑 1434 +库 1435 +荒 1436 +魅 1437 +鹤 1438 +售 1439 +承 1440 +忆 1441 +琪 1442 +舟 1443 +敬 1444 +痘 1445 +败 1446 +瘦 1447 +龄 1448 +胖 1449 +甘 1450 +叠 1451 +聚 1452 +训 1453 +删 1454 +笨 1455 +滩 1456 +澳 1457 +显 1458 +窗 1459 +倾 1460 +幽 1461 +厘 1462 +惑 1463 +挑 1464 +泪 1465 +螺 1466 +严 1467 +狱 1468 +亦 1469 +烤 1470 +晗 1471 +晴 1472 +驶 1473 +截 1474 +企 1475 +获 1476 +符 1477 +碰 1478 +丑 1479 +据 1480 +抢 1481 +醉 1482 +驰 1483 +私 1484 +停 1485 +席 1486 +此 1487 +映 1488 +迅 1489 +芝 1490 +翔 1491 +翼 1492 +纹 1493 +汪 1494 +她 1495 +贤 1496 +嫁 1497 +娇 1498 +氏 1499 +殖 1500 +悦 1501 +帽 1502 +壮 1503 +枝 1504 +饿 1505 +厉 1506 +饮 1507 +笼 1508 +寿 1509 +混 1510 +溜 1511 +唤 1512 +钠 1513 +醋 1514 +府 1515 +织 1516 +肝 1517 +赏 1518 +硫 1519 +宅 1520 +牧 1521 +厦 1522 +岩 1523 +焰 1524 +裂 1525 +霜 1526 +确 1527 +幕 1528 +召 1529 +翅 1530 +颗 1531 +润 1532 +蟹 1533 +涛 1534 +咋 1535 +辉 1536 +姜 1537 +差 1538 +婷 1539 +扣 1540 +仿 1541 +咳 1542 +斤 1543 +坚 1544 +抓 1545 +箭 1546 +赫 1547 +繁 1548 +宏 1549 +曾 1550 +淮 1551 +诱 1552 +厨 1553 +鳄 1554 +副 1555 +肌 1556 +葡 1557 +贾 1558 +菊 1559 +玺 1560 +硬 1561 +尺 1562 +墨 1563 +莎 1564 +轩 1565 +肿 1566 +览 1567 +潘 1568 +仓 1569 +萄 1570 +爹 1571 +借 1572 +即 1573 +搭 1574 +吐 1575 +页 1576 +哭 1577 +匆 1578 +乔 1579 +撸 1580 +乙 1581 +弯 1582 +沟 1583 +蛙 1584 +委 1585 +尖 1586 +塘 1587 +继 1588 +喂 1589 +悲 1590 +孝 1591 +暑 1592 +茄 1593 +纵 1594 +梯 1595 +绘 1596 +薇 1597 +宽 1598 +坛 1599 +勤 1600 +楚 1601 +坊 1602 +舌 1603 +穷 1604 +卧 1605 +鸣 1606 +乖 1607 +闲 1608 +饥 1609 +溶 1610 +醒 1611 +痒 1612 +宗 1613 +仇 1614 +咭 1615 +柔 1616 +施 1617 +亭 1618 +毫 1619 +豫 1620 +烊 1621 +飘 1622 +迎 1623 +议 1624 +酱 1625 +违 1626 +宣 1627 +淫 1628 +层 1629 +滚 1630 +辛 1631 +固 1632 +嫣 1633 +纱 1634 +膏 1635 +椒 1636 +姿 1637 +柏 1638 +宿 1639 +恒 1640 +柱 1641 +吊 1642 +捷 1643 +腺 1644 +孟 1645 +胞 1646 +徒 1647 +粒 1648 +或 1649 +哦 1650 +肾 1651 +诉 1652 +丫 1653 +霞 1654 +颈 1655 +萧 1656 +霍 1657 +汗 1658 +绩 1659 +残 1660 +豹 1661 +烂 1662 +枣 1663 +浦 1664 +桑 1665 +姨 1666 +蹦 1667 +齿 1668 +呃 1669 +尘 1670 +淋 1671 +薄 1672 +扇 1673 +嘟 1674 +棉 1675 +汁 1676 +辑 1677 +稀 1678 +培 1679 +疾 1680 +懒 1681 +漠 1682 +惜 1683 +著 1684 +锡 1685 +杂 1686 +玄 1687 +踩 1688 +咸 1689 +拌 1690 +革 1691 +莓 1692 +靠 1693 +菠 1694 +陀 1695 +峡 1696 +眠 1697 +绑 1698 +陕 1699 +撕 1700 +陶 1701 +菱 1702 +擦 1703 +葱 1704 +筋 1705 +并 1706 +盆 1707 +紧 1708 +哒 1709 +述 1710 +逗 1711 +档 1712 +媳 1713 +伏 1714 +忧 1715 +延 1716 +璃 1717 +辰 1718 +析 1719 +寂 1720 +涵 1721 +橡 1722 +荡 1723 +顿 1724 +域 1725 +缺 1726 +氯 1727 +屎 1728 +邻 1729 +遗 1730 +黎 1731 +翠 1732 +冻 1733 +菌 1734 +励 1735 +衡 1736 +穴 1737 +奋 1738 +涯 1739 +监 1740 +咖 1741 +狸 1742 +磨 1743 +董 1744 +疆 1745 +姚 1746 +玻 1747 +番 1748 +弃 1749 +哺 1750 +拖 1751 +赶 1752 +究 1753 +猛 1754 +摄 1755 +妮 1756 +芒 1757 +脏 1758 +咬 1759 +鸿 1760 +粘 1761 +它 1762 +研 1763 +聪 1764 +蕉 1765 +疹 1766 +薛 1767 +壳 1768 +鹏 1769 +订 1770 +摆 1771 +坑 1772 +彼 1773 +喷 1774 +逊 1775 +诞 1776 +卵 1777 +链 1778 +蔡 1779 +炉 1780 +彭 1781 +钙 1782 +杏 1783 +督 1784 +额 1785 +鞭 1786 +淡 1787 +葛 1788 +浮 1789 +膀 1790 +怡 1791 +涂 1792 +牡 1793 +焦 1794 +驴 1795 +鼎 1796 +距 1797 +累 1798 +浴 1799 +俏 1800 +稿 1801 +缩 1802 +割 1803 +腔 1804 +宙 1805 +忠 1806 +党 1807 +筒 1808 +橙 1809 +馨 1810 +须 1811 +芽 1812 +晋 1813 +竞 1814 +琳 1815 +娱 1816 +骚 1817 +嗽 1818 +避 1819 +筝 1820 +懂 1821 +寸 1822 +蒂 1823 +穹 1824 +摘 1825 +乃 1826 +萍 1827 +蝎 1828 +互 1829 +诸 1830 +御 1831 +窝 1832 +巾 1833 +瓷 1834 +坡 1835 +阜 1836 +魏 1837 +隋 1838 +腊 1839 +矿 1840 +肯 1841 +旺 1842 +扑 1843 +厚 1844 +判 1845 +闭 1846 +莹 1847 +韵 1848 +迁 1849 +阵 1850 +烦 1851 +惹 1852 +癌 1853 +嘛 1854 +浏 1855 +脂 1856 +蒲 1857 +茅 1858 +柜 1859 +貌 1860 +妍 1861 +啪 1862 +绳 1863 +蒜 1864 +沧 1865 +弱 1866 +筑 1867 +疑 1868 +犯 1869 +磁 1870 +淇 1871 +执 1872 +陌 1873 +泊 1874 +逸 1875 +垃 1876 +误 1877 +竖 1878 +侧 1879 +渡 1880 +潜 1881 +辞 1882 +芬 1883 +弊 1884 +忙 1885 +罐 1886 +碎 1887 +罩 1888 +熙 1889 +圾 1890 +袭 1891 +困 1892 +叮 1893 +俄 1894 +寄 1895 +勒 1896 +荆 1897 +戚 1898 +岸 1899 +铺 1900 +束 1901 +妞 1902 +融 1903 +咏 1904 +剂 1905 +朴 1906 +撒 1907 +甄 1908 +谚 1909 +亿 1910 +辆 1911 +杆 1912 +蓉 1913 +庐 1914 +浆 1915 +鸽 1916 +蒋 1917 +粗 1918 +耶 1919 +添 1920 +翁 1921 +蚂 1922 +垂 1923 +踪 1924 +荐 1925 +辨 1926 +蘑 1927 +蚁 1928 +漆 1929 +笙 1930 +策 1931 +慰 1932 +沂 1933 +但 1934 +柿 1935 +潭 1936 +夹 1937 +撞 1938 +刃 1939 +仰 1940 +阁 1941 +毅 1942 +郁 1943 +蕾 1944 +乎 1945 +愁 1946 +轿 1947 +吕 1948 +昨 1949 +椎 1950 +脉 1951 +巡 1952 +氨 1953 +烫 1954 +俱 1955 +岗 1956 +返 1957 +振 1958 +祸 1959 +串 1960 +蚕 1961 +勃 1962 +丛 1963 +莞 1964 +赢 1965 +谊 1966 +粥 1967 +帆 1968 +访 1969 +厕 1970 +诀 1971 +骂 1972 +奈 1973 +鉴 1974 +肺 1975 +辫 1976 +骏 1977 +葵 1978 +迟 1979 +赚 1980 +慈 1981 +函 1982 +铅 1983 +献 1984 +挡 1985 +驼 1986 +择 1987 +触 1988 +袁 1989 +晕 1990 +籽 1991 +冯 1992 +棍 1993 +协 1994 +邦 1995 +爬 1996 +糊 1997 +仲 1998 +桐 1999 +菩 2000 +郸 2001 +薯 2002 +够 2003 +径 2004 +税 2005 +斩 2006 +塑 2007 +胀 2008 +赌 2009 +锐 2010 +氓 2011 +隔 2012 +灿 2013 +浓 2014 +牵 2015 +碱 2016 +绒 2017 +肃 2018 +券 2019 +阔 2020 +檬 2021 +梭 2022 +丈 2023 +劲 2024 +啡 2025 +悬 2026 +坤 2027 +馅 2028 +瑜 2029 +呆 2030 +勿 2031 +虐 2032 +栏 2033 +哲 2034 +殿 2035 +缓 2036 +扩 2037 +恢 2038 +凶 2039 +纷 2040 +箫 2041 +讨 2042 +矮 2043 +碗 2044 +披 2045 +袖 2046 +嫂 2047 +械 2048 +侯 2049 +丧 2050 +责 2051 +噜 2052 +闺 2053 +茎 2054 +屠 2055 +夸 2056 +拥 2057 +扰 2058 +骤 2059 +筷 2060 +占 2061 +旦 2062 +屈 2063 +寞 2064 +贷 2065 +畅 2066 +媛 2067 +匹 2068 +暮 2069 +吨 2070 +邯 2071 +衫 2072 +渔 2073 +粮 2074 +贸 2075 +悄 2076 +灾 2077 +煎 2078 +唇 2079 +骗 2080 +逢 2081 +恨 2082 +淑 2083 +患 2084 +兑 2085 +惯 2086 +迈 2087 +卢 2088 +慕 2089 +恰 2090 +贫 2091 +阻 2092 +供 2093 +颂 2094 +咚 2095 +纲 2096 +蜀 2097 +蜗 2098 +掘 2099 +茂 2100 +储 2101 +藤 2102 +扁 2103 +麟 2104 +柠 2105 +骆 2106 +崇 2107 +泸 2108 +丢 2109 +螃 2110 +疮 2111 +享 2112 +秒 2113 +铝 2114 +仗 2115 +彤 2116 +恭 2117 +吾 2118 +贞 2119 +衰 2120 +斜 2121 +喉 2122 +晒 2123 +雁 2124 +穆 2125 +搬 2126 +磊 2127 +拆 2128 +悔 2129 +卿 2130 +浅 2131 +涨 2132 +硝 2133 +滋 2134 +崛 2135 +珊 2136 +茉 2137 +弓 2138 +湘 2139 +枫 2140 +踏 2141 +榴 2142 +寨 2143 +翰 2144 +吟 2145 +均 2146 +蝉 2147 +卸 2148 +嘎 2149 +愚 2150 +剖 2151 +哎 2152 +污 2153 +镖 2154 +遮 2155 +垫 2156 +勾 2157 +鸦 2158 +鲤 2159 +埃 2160 +替 2161 +忌 2162 +犹 2163 +劫 2164 +笛 2165 +咤 2166 +傅 2167 +绕 2168 +帐 2169 +鹦 2170 +狠 2171 +缝 2172 +遍 2173 +拾 2174 +巢 2175 +僧 2176 +煤 2177 +铭 2178 +厌 2179 +鹉 2180 +帕 2181 +伞 2182 +蹄 2183 +阶 2184 +佐 2185 +既 2186 +抚 2187 +吞 2188 +橘 2189 +跨 2190 +肩 2191 +摔 2192 +猩 2193 +凝 2194 +甫 2195 +旭 2196 +赖 2197 +瘤 2198 +羞 2199 +叹 2200 +弦 2201 +璐 2202 +臣 2203 +沃 2204 +忽 2205 +闷 2206 +钩 2207 +嬛 2208 +谋 2209 +豌 2210 +骄 2211 +昂 2212 +耐 2213 +鑫 2214 +审 2215 +另 2216 +呐 2217 +帘 2218 +舰 2219 +咒 2220 +栀 2221 +崔 2222 +顷 2223 +跃 2224 +罚 2225 +谎 2226 +奉 2227 +钉 2228 +瀑 2229 +障 2230 +诵 2231 +挥 2232 +驱 2233 +蟒 2234 +渴 2235 +蛮 2236 +枕 2237 +猿 2238 +砖 2239 +瑶 2240 +援 2241 +咙 2242 +稻 2243 +潇 2244 +孽 2245 +拨 2246 +刮 2247 +奴 2248 +辅 2249 +催 2250 +飙 2251 +蓬 2252 +腌 2253 +届 2254 +谣 2255 +踢 2256 +笋 2257 +陪 2258 +壶 2259 +且 2260 +姬 2261 +渐 2262 +予 2263 +幅 2264 +倩 2265 +椅 2266 +芙 2267 +逍 2268 +坠 2269 +驻 2270 +侵 2271 +濮 2272 +毁 2273 +鲸 2274 +靓 2275 +钾 2276 +洒 2277 +栽 2278 +伽 2279 +侏 2280 +爪 2281 +嘻 2282 +杞 2283 +允 2284 +楠 2285 +践 2286 +扶 2287 +轨 2288 +巫 2289 +谦 2290 +昏 2291 +邢 2292 +哑 2293 +爵 2294 +抹 2295 +苑 2296 +硕 2297 +漏 2298 +寡 2299 +啤 2300 +伙 2301 +扒 2302 +殊 2303 +尤 2304 +沫 2305 +贡 2306 +鞍 2307 +蚊 2308 +贪 2309 +蝠 2310 +藕 2311 +冈 2312 +兆 2313 +萱 2314 +梳 2315 +灌 2316 +仆 2317 +皆 2318 +咽 2319 +粱 2320 +蝙 2321 +襄 2322 +弗 2323 +缸 2324 +禹 2325 +咕 2326 +菏 2327 +您 2328 +宰 2329 +斋 2330 +慌 2331 +絮 2332 +痕 2333 +崖 2334 +韭 2335 +担 2336 +锈 2337 +黛 2338 +盈 2339 +稳 2340 +翩 2341 +呱 2342 +疏 2343 +嫩 2344 +轴 2345 +耽 2346 +熬 2347 +怨 2348 +盲 2349 +脾 2350 +详 2351 +却 2352 +顽 2353 +鲍 2354 +疫 2355 +溃 2356 +茫 2357 +闰 2358 +贱 2359 +鱿 2360 +扭 2361 +舅 2362 +痣 2363 +劝 2364 +浑 2365 +廷 2366 +迫 2367 +驯 2368 +挣 2369 +耗 2370 +碑 2371 +饶 2372 +雯 2373 +卑 2374 +锻 2375 +辈 2376 +哇 2377 +廊 2378 +契 2379 +赠 2380 +艇 2381 +辩 2382 +努 2383 +枸 2384 +枯 2385 +芜 2386 +绸 2387 +碟 2388 +潍 2389 +逐 2390 +咯 2391 +框 2392 +跆 2393 +竟 2394 +挺 2395 +娟 2396 +挠 2397 +尝 2398 +咱 2399 +蜓 2400 +赋 2401 +麒 2402 +姻 2403 +臂 2404 +拐 2405 +敲 2406 +牢 2407 +损 2408 +誉 2409 +祭 2410 +捡 2411 +荟 2412 +砍 2413 +乾 2414 +墅 2415 +砂 2416 +蜻 2417 +烛 2418 +婉 2419 +脖 2420 +饱 2421 +奕 2422 +吓 2423 +煲 2424 +授 2425 +耍 2426 +叙 2427 +斌 2428 +欺 2429 +呦 2430 +拽 2431 +橱 2432 +肛 2433 +舔 2434 +盾 2435 +梗 2436 +獒 2437 +遵 2438 +媒 2439 +秧 2440 +疲 2441 +勋 2442 +夷 2443 +陷 2444 +恼 2445 +睿 2446 +苔 2447 +彬 2448 +贯 2449 +榆 2450 +弥 2451 +茜 2452 +璧 2453 +馒 2454 +乒 2455 +诊 2456 +愈 2457 +棵 2458 +盼 2459 +兮 2460 +窖 2461 +霹 2462 +渊 2463 +嫡 2464 +歪 2465 +堆 2466 +胥 2467 +喵 2468 +雳 2469 +痔 2470 +醇 2471 +芹 2472 +杠 2473 +辱 2474 +裳 2475 +乓 2476 +虑 2477 +谐 2478 +薪 2479 +瑟 2480 +剩 2481 +戈 2482 +岚 2483 +锥 2484 +澜 2485 +逝 2486 +涩 2487 +敦 2488 +俪 2489 +姥 2490 +缠 2491 +揭 2492 +翘 2493 +轼 2494 +沁 2495 +吵 2496 +臀 2497 +琼 2498 +泼 2499 +挤 2500 +仑 2501 +惨 2502 +喽 2503 +姗 2504 +曝 2505 +韦 2506 +堰 2507 +辕 2508 +敷 2509 +兹 2510 +伪 2511 +纤 2512 +蚌 2513 +琦 2514 +淄 2515 +役 2516 +脊 2517 +匪 2518 +嗓 2519 +哀 2520 +豚 2521 +冤 2522 +衬 2523 +掩 2524 +措 2525 +卤 2526 +竿 2527 +钰 2528 +宴 2529 +稣 2530 +疡 2531 +擎 2532 +悉 2533 +喊 2534 +嘿 2535 +痴 2536 +捏 2537 +糯 2538 +廉 2539 +淀 2540 +狄 2541 +纽 2542 +娥 2543 +魁 2544 +犀 2545 +霄 2546 +彦 2547 +卦 2548 +祛 2549 +屯 2550 +裕 2551 +涕 2552 +邱 2553 +哮 2554 +础 2555 +邵 2556 +棱 2557 +厢 2558 +岷 2559 +屌 2560 +伐 2561 +浇 2562 +冥 2563 +巩 2564 +嗨 2565 +泄 2566 +炭 2567 +攀 2568 +沸 2569 +呵 2570 +痰 2571 +堵 2572 +榨 2573 +虽 2574 +悍 2575 +溺 2576 +甸 2577 +娶 2578 +拒 2579 +艰 2580 +泻 2581 +靖 2582 +萤 2583 +烽 2584 +燥 2585 +妓 2586 +赣 2587 +郊 2588 +锤 2589 +棠 2590 +煌 2591 +抛 2592 +脆 2593 +匠 2594 +鞠 2595 +焖 2596 +筹 2597 +遭 2598 +魄 2599 +鸳 2600 +喇 2601 +轰 2602 +抖 2603 +瞳 2604 +遂 2605 +欠 2606 +泌 2607 +瓣 2608 +梓 2609 +娅 2610 +拇 2611 +拯 2612 +埋 2613 +棚 2614 +鸯 2615 +循 2616 +怜 2617 +袍 2618 +昭 2619 +兼 2620 +挨 2621 +瘩 2622 +昕 2623 +硅 2624 +咨 2625 +抑 2626 +栗 2627 +呛 2628 +榄 2629 +滕 2630 +蔷 2631 +漯 2632 +疙 2633 +粑 2634 +搏 2635 +沿 2636 +握 2637 +巷 2638 +葬 2639 +螂 2640 +禽 2641 +膝 2642 +邑 2643 +绞 2644 +莺 2645 +峨 2646 +峙 2647 +俺 2648 +鄙 2649 +焊 2650 +倚 2651 +叭 2652 +酥 2653 +株 2654 +捞 2655 +倪 2656 +愉 2657 +瘾 2658 +绥 2659 +翡 2660 +脐 2661 +促 2662 +巅 2663 +抵 2664 +粤 2665 +俩 2666 +瞬 2667 +禾 2668 +蓄 2669 +曰 2670 +溢 2671 +绪 2672 +丙 2673 +沾 2674 +皂 2675 +酿 2676 +陋 2677 +绰 2678 +喘 2679 +椰 2680 +耻 2681 +裔 2682 +锌 2683 +儒 2684 +柚 2685 +惩 2686 +埠 2687 +囧 2688 +畏 2689 +佑 2690 +邹 2691 +灸 2692 +薰 2693 +坪 2694 +崩 2695 +肪 2696 +崎 2697 +漳 2698 +昵 2699 +酮 2700 +橄 2701 +鹃 2702 +呕 2703 +粽 2704 +猕 2705 +茹 2706 +峻 2707 +唉 2708 +呈 2709 +唢 2710 +佣 2711 +汕 2712 +削 2713 +恍 2714 +霖 2715 +梧 2716 +携 2717 +惧 2718 +誓 2719 +嗝 2720 +庞 2721 +曦 2722 +泗 2723 +沌 2724 +湛 2725 +瞎 2726 +暇 2727 +镯 2728 +啸 2729 +栩 2730 +窟 2731 +涉 2732 +鲫 2733 +伸 2734 +倦 2735 +谓 2736 +沐 2737 +刹 2738 +貂 2739 +耕 2740 +诡 2741 +氮 2742 +聋 2743 +淳 2744 +弘 2745 +霾 2746 +躲 2747 +佟 2748 +藻 2749 +纺 2750 +淹 2751 +靴 2752 +狭 2753 +槽 2754 +涌 2755 +庸 2756 +娲 2757 +谍 2758 +桔 2759 +磷 2760 +捣 2761 +焉 2762 +砸 2763 +赔 2764 +鳞 2765 +甩 2766 +渣 2767 +芈 2768 +债 2769 +蔚 2770 +庶 2771 +糟 2772 +篱 2773 +凸 2774 +睫 2775 +拘 2776 +挫 2777 +屑 2778 +哩 2779 +诛 2780 +跤 2781 +啼 2782 +蠢 2783 +栋 2784 +窦 2785 +哟 2786 +灶 2787 +皓 2788 +凭 2789 +晏 2790 +逛 2791 +辐 2792 +铲 2793 +辟 2794 +窍 2795 +俐 2796 +钏 2797 +坝 2798 +蝌 2799 +饲 2800 +堪 2801 +蚪 2802 +凿 2803 +乞 2804 +矩 2805 +鳅 2806 +泣 2807 +镁 2808 +颠 2809 +劈 2810 +珀 2811 +矛 2812 +娄 2813 +氟 2814 +扔 2815 +皱 2816 +窃 2817 +裹 2818 +渠 2819 +哄 2820 +鹊 2821 +褒 2822 +绯 2823 +汝 2824 +叽 2825 +剥 2826 +孜 2827 +楂 2828 +粪 2829 +愧 2830 +斧 2831 +锯 2832 +仕 2833 +昔 2834 +阑 2835 +晰 2836 +缤 2837 +孢 2838 +羚 2839 +抬 2840 +屿 2841 +澄 2842 +撑 2843 +酵 2844 +糜 2845 +鄂 2846 +飓 2847 +芯 2848 +薏 2849 +渭 2850 +昊 2851 +虞 2852 +疤 2853 +暂 2854 +韶 2855 +惟 2856 +熏 2857 +坟 2858 +畔 2859 +澈 2860 +伍 2861 +坞 2862 +篷 2863 +慎 2864 +躺 2865 +衩 2866 +茨 2867 +嫦 2868 +凳 2869 +钧 2870 +馗 2871 +竭 2872 +孵 2873 +俯 2874 +籁 2875 +腥 2876 +躬 2877 +汾 2878 +琵 2879 +咆 2880 +偿 2881 +胳 2882 +荔 2883 +芋 2884 +蔽 2885 +贬 2886 +琶 2887 +羡 2888 +旷 2889 +炅 2890 +苯 2891 +酶 2892 +闫 2893 +稍 2894 +阎 2895 +尹 2896 +螳 2897 +邀 2898 +骐 2899 +膊 2900 +颁 2901 +鹂 2902 +帖 2903 +涡 2904 +兜 2905 +缅 2906 +嘞 2907 +躁 2908 +祁 2909 +茵 2910 +估 2911 +郝 2912 +妒 2913 +秤 2914 +焚 2915 +耿 2916 +勉 2917 +抠 2918 +晃 2919 +彪 2920 +滤 2921 +莽 2922 +晖 2923 +瞧 2924 +履 2925 +禅 2926 +矣 2927 +伶 2928 +碘 2929 +懈 2930 +彻 2931 +甚 2932 +葩 2933 +珑 2934 +袄 2935 +娆 2936 +驹 2937 +敞 2938 +唑 2939 +馄 2940 +滥 2941 +腚 2942 +滔 2943 +肇 2944 +仅 2945 +栓 2946 +纬 2947 +囚 2948 +棕 2949 +琅 2950 +韬 2951 +雌 2952 +詹 2953 +凹 2954 +掏 2955 +樊 2956 +抒 2957 +叛 2958 +卉 2959 +跌 2960 +朔 2961 +萎 2962 +俑 2963 +嚣 2964 +赴 2965 +冉 2966 +剃 2967 +哗 2968 +捐 2969 +搅 2970 +饨 2971 +宪 2972 +缕 2973 +颤 2974 +刁 2975 +趾 2976 +渤 2977 +寇 2978 +蛤 2979 +蝇 2980 +毯 2981 +荨 2982 +檀 2983 +摊 2984 +锰 2985 +绅 2986 +殷 2987 +颐 2988 +沥 2989 +捆 2990 +旱 2991 +釜 2992 +龈 2993 +拓 2994 +洱 2995 +勺 2996 +癫 2997 +泛 2998 +衢 2999 +寝 3000 +跪 3001 +隶 3002 +槐 3003 +鸥 3004 +妄 3005 +歉 3006 +亨 3007 +沽 3008 +揉 3009 +簧 3010 +泵 3011 +嚼 3012 +猬 3013 +癣 3014 +肘 3015 +婶 3016 +蜈 3017 +亏 3018 +滁 3019 +恤 3020 +嗜 3021 +瀚 3022 +呗 3023 +汹 3024 +羹 3025 +稚 3026 +蕊 3027 +蕴 3028 +俭 3029 +禧 3030 +妾 3031 +亩 3032 +枭 3033 +煞 3034 +芸 3035 +镐 3036 +憋 3037 +苟 3038 +窥 3039 +掠 3040 +纠 3041 +栈 3042 +酚 3043 +鹭 3044 +莆 3045 +栖 3046 +胰 3047 +辙 3048 +嫉 3049 +蚣 3050 +尧 3051 +朦 3052 +乏 3053 +沪 3054 +慨 3055 +庚 3056 +诈 3057 +昧 3058 +疱 3059 +弈 3060 +烷 3061 +龚 3062 +漾 3063 +羲 3064 +谨 3065 +撇 3066 +澎 3067 +侍 3068 +佰 3069 +浊 3070 +窄 3071 +讽 3072 +蔗 3073 +沛 3074 +羯 3075 +凛 3076 +噬 3077 +吁 3078 +雍 3079 +腻 3080 +矫 3081 +涧 3082 +箍 3083 +祺 3084 +崽 3085 +秃 3086 +罕 3087 +蔻 3088 +癜 3089 +塌 3090 +纶 3091 +烯 3092 +胧 3093 +恳 3094 +奎 3095 +嵩 3096 +碌 3097 +蔼 3098 +鞘 3099 +烙 3100 +亥 3101 +啄 3102 +瞻 3103 +仍 3104 +酯 3105 +浣 3106 +荧 3107 +焕 3108 +畜 3109 +椿 3110 +拙 3111 +璨 3112 +盔 3113 +隧 3114 +押 3115 +肢 3116 +廖 3117 +拂 3118 +瑕 3119 +渺 3120 +璀 3121 +婿 3122 +匀 3123 +蔓 3124 +腕 3125 +骷 3126 +梆 3127 +哔 3128 +腋 3129 +樟 3130 +锣 3131 +矢 3132 +碍 3133 +瘙 3134 +茸 3135 +潢 3136 +潼 3137 +祈 3138 +溴 3139 +嫌 3140 +脓 3141 +杖 3142 +漓 3143 +埔 3144 +瑚 3145 +衷 3146 +闽 3147 +蚓 3148 +歧 3149 +丞 3150 +郡 3151 +汀 3152 +睁 3153 +肆 3154 +撼 3155 +荫 3156 +蹲 3157 +灼 3158 +朽 3159 +豁 3160 +撤 3161 +馍 3162 +蚯 3163 +琉 3164 +亢 3165 +捧 3166 +烁 3167 +阱 3168 +隙 3169 +溅 3170 +棺 3171 +熔 3172 +劣 3173 +磅 3174 +铛 3175 +趁 3176 +呜 3177 +雏 3178 +旨 3179 +俞 3180 +钡 3181 +稽 3182 +冀 3183 +宛 3184 +璇 3185 +巍 3186 +枚 3187 +沭 3188 +刊 3189 +奢 3190 +擅 3191 +诲 3192 +搓 3193 +渍 3194 +咔 3195 +垣 3196 +奄 3197 +丐 3198 +趋 3199 +蟾 3200 +衔 3201 +禄 3202 +瓢 3203 +瞿 3204 +苞 3205 +桨 3206 +髅 3207 +趴 3208 +凄 3209 +绽 3210 +噩 3211 +狡 3212 +嚏 3213 +鳝 3214 +琐 3215 +拢 3216 +敖 3217 +韧 3218 +骇 3219 +壹 3220 +镑 3221 +稼 3222 +搁 3223 +棘 3224 +闸 3225 +蛰 3226 +哨 3227 +猥 3228 +纨 3229 +笆 3230 +淤 3231 +亳 3232 +鲈 3233 +倘 3234 +疣 3235 +擒 3236 +跷 3237 +蟆 3238 +楷 3239 +妲 3240 +嘲 3241 +瑙 3242 +冶 3243 +壤 3244 +蜒 3245 +莴 3246 +婵 3247 +玖 3248 +苓 3249 +戎 3250 +喧 3251 +蜥 3252 +弧 3253 +垢 3254 +酪 3255 +桦 3256 +喱 3257 +哼 3258 +髓 3259 +阀 3260 +忻 3261 +牲 3262 +拱 3263 +噪 3264 +赐 3265 +谅 3266 +兀 3267 +胚 3268 +粟 3269 +蜿 3270 +缉 3271 +绮 3272 +蛾 3273 +哉 3274 +嬉 3275 +窑 3276 +疸 3277 +钦 3278 +倔 3279 +岂 3280 +鼹 3281 +喋 3282 +玮 3283 +贩 3284 +蛹 3285 +芪 3286 +佬 3287 +酬 3288 +缪 3289 +汶 3290 +裆 3291 +枉 3292 +聂 3293 +腮 3294 +侨 3295 +甥 3296 +昼 3297 +钝 3298 +娩 3299 +暨 3300 +炊 3301 +凋 3302 +峪 3303 +嫖 3304 +羔 3305 +筱 3306 +磕 3307 +瞪 3308 +芥 3309 +陛 3310 +刨 3311 +蚀 3312 +蒿 3313 +耸 3314 +胺 3315 +艘 3316 +粼 3317 +萋 3318 +吼 3319 +烹 3320 +溧 3321 +钗 3322 +喔 3323 +筛 3324 +倡 3325 +谏 3326 +黔 3327 +锄 3328 +蟀 3329 +炳 3330 +琥 3331 +驿 3332 +睾 3333 +衍 3334 +椭 3335 +蟋 3336 +瘫 3337 +荚 3338 +渝 3339 +峦 3340 +眨 3341 +襟 3342 +熹 3343 +锵 3344 +傍 3345 +绔 3346 +缴 3347 +堤 3348 +醛 3349 +涟 3350 +傣 3351 +荞 3352 +觅 3353 +慷 3354 +挽 3355 +斛 3356 +靡 3357 +烘 3358 +浸 3359 +瘁 3360 +泾 3361 +蟑 3362 +榕 3363 +藿 3364 +祷 3365 +琢 3366 +褐 3367 +嗡 3368 +曙 3369 +蜇 3370 +妊 3371 +驭 3372 +辖 3373 +吱 3374 +遐 3375 +痧 3376 +笠 3377 +炙 3378 +擀 3379 +讶 3380 +咧 3381 +鲅 3382 +扯 3383 +缇 3384 +趟 3385 +嘱 3386 +秉 3387 +颇 3388 +苇 3389 +憾 3390 +恙 3391 +拦 3392 +钞 3393 +磐 3394 +郴 3395 +峭 3396 +攘 3397 +挪 3398 +痱 3399 +剿 3400 +陇 3401 +稠 3402 +坂 3403 +叨 3404 +牺 3405 +晟 3406 +吝 3407 +殇 3408 +匿 3409 +陨 3410 +茧 3411 +绚 3412 +蝈 3413 +裴 3414 +闵 3415 +缆 3416 +梢 3417 +盏 3418 +诶 3419 +鸠 3420 +摧 3421 +铸 3422 +掀 3423 +嘀 3424 +鳖 3425 +垒 3426 +锲 3427 +扳 3428 +揽 3429 +沦 3430 +坎 3431 +桩 3432 +竽 3433 +囵 3434 +彰 3435 +狩 3436 +阡 3437 +袅 3438 +尬 3439 +蛊 3440 +呻 3441 +铂 3442 +膨 3443 +舜 3444 +庵 3445 +槟 3446 +霓 3447 +辘 3448 +镶 3449 +磺 3450 +凑 3451 +痞 3452 +芷 3453 +匕 3454 +唧 3455 +喳 3456 +嗦 3457 +斥 3458 +漩 3459 +逵 3460 +炯 3461 +牟 3462 +岐 3463 +穗 3464 +梵 3465 +逮 3466 +萃 3467 +芮 3468 +尴 3469 +煸 3470 +呸 3471 +阙 3472 +氛 3473 +蹑 3474 +鹌 3475 +鸵 3476 +胭 3477 +鹑 3478 +枇 3479 +璋 3480 +蜴 3481 +囫 3482 +舂 3483 +祠 3484 +糙 3485 +恬 3486 +悯 3487 +汰 3488 +尉 3489 +纫 3490 +彝 3491 +挚 3492 +淌 3493 +盎 3494 +羿 3495 +蓟 3496 +竺 3497 +瞩 3498 +杷 3499 +昙 3500 +朕 3501 +惶 3502 +惰 3503 +婕 3504 +傀 3505 +炽 3506 +儡 3507 +簇 3508 +绎 3509 +秽 3510 +弛 3511 +迂 3512 +怦 3513 +罢 3514 +腩 3515 +喰 3516 +蜕 3517 +荠 3518 +啬 3519 +逻 3520 +弩 3521 +榻 3522 +婺 3523 +迢 3524 +暧 3525 +犊 3526 +筐 3527 +烨 3528 +鲟 3529 +惭 3530 +涿 3531 +毙 3532 +涣 3533 +鲶 3534 +痫 3535 +诅 3536 +沼 3537 +黏 3538 +禺 3539 +虱 3540 +卒 3541 +喀 3542 +淅 3543 +皑 3544 +堕 3545 +黯 3546 +咐 3547 +墩 3548 +剁 3549 +莘 3550 +擂 3551 +簪 3552 +睹 3553 +斓 3554 +缭 3555 +犁 3556 +肋 3557 +挎 3558 +挞 3559 +崭 3560 +夭 3561 +厄 3562 +蹭 3563 +妥 3564 +圃 3565 +唠 3566 +遣 3567 +茯 3568 +泷 3569 +徊 3570 +砰 3571 +辜 3572 +饪 3573 +秩 3574 +帷 3575 +帜 3576 +铵 3577 +菁 3578 +俘 3579 +旬 3580 +痿 3581 +叼 3582 +褪 3583 +眺 3584 +徘 3585 +屡 3586 +檐 3587 +猾 3588 +戳 3589 +赎 3590 +肴 3591 +逅 3592 +奂 3593 +橇 3594 +皋 3595 +岔 3596 +邂 3597 +芊 3598 +瑾 3599 +茗 3600 +骥 3601 +睦 3602 +淼 3603 +滦 3604 +砌 3605 +邳 3606 +胁 3607 +驸 3608 +侄 3609 +署 3610 +尻 3611 +蝗 3612 +癞 3613 +剔 3614 +汐 3615 +冕 3616 +歼 3617 +仞 3618 +蚱 3619 +翟 3620 +湄 3621 +寐 3622 +睢 3623 +茬 3624 +炬 3625 +懿 3626 +掰 3627 +琊 3628 +陡 3629 +镰 3630 +鬓 3631 +腱 3632 +僻 3633 +绢 3634 +墟 3635 +孺 3636 +娓 3637 +毽 3638 +绊 3639 +馈 3640 +皖 3641 +跋 3642 +啃 3643 +珂 3644 +怯 3645 +饵 3646 +蛟 3647 +寥 3648 +孚 3649 +峥 3650 +阮 3651 +撩 3652 +馋 3653 +嘘 3654 +杵 3655 +恕 3656 +诫 3657 +揍 3658 +嶙 3659 +胯 3660 +粹 3661 +眩 3662 +蚝 3663 +煜 3664 +诧 3665 +鳕 3666 +垠 3667 +娼 3668 +铢 3669 +伺 3670 +瘟 3671 +蔺 3672 +胱 3673 +虏 3674 +烩 3675 +娴 3676 +飒 3677 +惚 3678 +涅 3679 +峋 3680 +雇 3681 +琛 3682 +窕 3683 +懊 3684 +晾 3685 +洙 3686 +屹 3687 +馁 3688 +苣 3689 +掐 3690 +眯 3691 +柄 3692 +濒 3693 +嚷 3694 +鳌 3695 +窈 3696 +钛 3697 +栾 3698 +窜 3699 +婊 3700 +榔 3701 +疝 3702 +裘 3703 +蓓 3704 +幄 3705 +霏 3706 +捶 3707 +郫 3708 +蘸 3709 +眷 3710 +颍 3711 +脯 3712 +挛 3713 +鹳 3714 +钮 3715 +嗒 3716 +糠 3717 +桓 3718 +骼 3719 +噼 3720 +葚 3721 +抉 3722 +侮 3723 +匡 3724 +洼 3725 +咀 3726 +惫 3727 +眸 3728 +婪 3729 +忐 3730 +蛐 3731 +芍 3732 +拧 3733 +莒 3734 +蚤 3735 +悚 3736 +坻 3737 +矾 3738 +忑 3739 +悼 3740 +壑 3741 +肽 3742 +蓑 3743 +攒 3744 +玟 3745 +鲢 3746 +歹 3747 +吭 3748 +痉 3749 +蛎 3750 +骁 3751 +娠 3752 +罂 3753 +缚 3754 +捅 3755 +瘸 3756 +舱 3757 +遁 3758 +寅 3759 +雹 3760 +珞 3761 +簸 3762 +酌 3763 +枢 3764 +蕨 3765 +簿 3766 +偃 3767 +隽 3768 +褂 3769 +轲 3770 +缀 3771 +钳 3772 +颊 3773 +戮 3774 +嵊 3775 +掷 3776 +涤 3777 +侃 3778 +嫔 3779 +瞄 3780 +膛 3781 +赁 3782 +癖 3783 +讳 3784 +皎 3785 +忡 3786 +隍 3787 +膳 3788 +搔 3789 +讼 3790 +唰 3791 +滟 3792 +冽 3793 +孰 3794 +兢 3795 +漱 3796 +躯 3797 +雎 3798 +嘹 3799 +捂 3800 +婧 3801 +蔑 3802 +眈 3803 +糗 3804 +镀 3805 +兖 3806 +鬟 3807 +唾 3808 +刿 3809 +羌 3810 +畸 3811 +涮 3812 +迭 3813 +榛 3814 +熄 3815 +礴 3816 +殴 3817 +熠 3818 +擞 3819 +矶 3820 +茴 3821 +颅 3822 +瓯 3823 +鳗 3824 +匣 3825 +滞 3826 +廓 3827 +痦 3828 +邋 3829 +戊 3830 +侈 3831 +惋 3832 +骅 3833 +岖 3834 +嶂 3835 +搂 3836 +敛 3837 +燎 3838 +砚 3839 +吏 3840 +椟 3841 +渲 3842 +螨 3843 +蛀 3844 +遢 3845 +噢 3846 +毓 3847 +蹊 3848 +锂 3849 +沱 3850 +崴 3851 +鹬 3852 +乍 3853 +奠 3854 +庾 3855 +焯 3856 +啰 3857 +鸢 3858 +扉 3859 +霎 3860 +耙 3861 +茁 3862 +懦 3863 +阆 3864 +柬 3865 +妨 3866 +矜 3867 +靶 3868 +蜃 3869 +湃 3870 +驳 3871 +唳 3872 +佗 3873 +崂 3874 +浜 3875 +舵 3876 +募 3877 +骛 3878 +瘀 3879 +馏 3880 +扛 3881 +槿 3882 +屄 3883 +蹬 3884 +茱 3885 +眶 3886 +屉 3887 +嚎 3888 +疚 3889 +涸 3890 +歆 3891 +啵 3892 +猝 3893 +藓 3894 +骊 3895 +胤 3896 +渗 3897 +铮 3898 +憧 3899 +鞅 3900 +颓 3901 +礁 3902 +悴 3903 +憬 3904 +嘚 3905 +曳 3906 +踌 3907 +酰 3908 +帚 3909 +墉 3910 +嵌 3911 +楞 3912 +砀 3913 +铿 3914 +喃 3915 +揣 3916 +筠 3917 +潋 3918 +曜 3919 +吩 3920 +徙 3921 +岱 3922 +缎 3923 +拷 3924 +憎 3925 +谛 3926 +铉 3927 +躇 3928 +坨 3929 +憔 3930 +觑 3931 +铡 3932 +紊 3933 +腑 3934 +唏 3935 +硒 3936 +撅 3937 +濡 3938 +苷 3939 +殡 3940 +妩 3941 +鹫 3942 +嗪 3943 +柑 3944 +斐 3945 +聆 3946 +郓 3947 +衙 3948 +浚 3949 +匈 3950 +涞 3951 +潺 3952 +戬 3953 +罔 3954 +坷 3955 +樵 3956 +荤 3957 +涓 3958 +酝 3959 +蹋 3960 +蠡 3961 +逾 3962 +掺 3963 +纣 3964 +弑 3965 +嗷 3966 +貔 3967 +钊 3968 +嬷 3969 +炝 3970 +镂 3971 +谴 3972 +忏 3973 +篆 3974 +砺 3975 +嘶 3976 +瞒 3977 +潦 3978 +狞 3979 +胗 3980 +妣 3981 +脍 3982 +呲 3983 +狈 3984 +垄 3985 +咩 3986 +怠 3987 +翱 3988 +瓮 3989 +掖 3990 +骞 3991 +愣 3992 +貅 3993 +盯 3994 +惬 3995 +驮 3996 +讥 3997 +噗 3998 +戟 3999 +亵 4000 +饯 4001 +葭 4002 +颦 4003 +卯 4004 +踵 4005 +臻 4006 +泓 4007 +肮 4008 +茼 4009 +浔 4010 +荥 4011 +鄞 4012 +洽 4013 +嗅 4014 +碾 4015 +疵 4016 +澹 4017 +绷 4018 +涪 4019 +桢 4020 +垦 4021 +璞 4022 +叱 4023 +耒 4024 +瑛 4025 +炕 4026 +沮 4027 +潸 4028 +蒹 4029 +婢 4030 +麝 4031 +攸 4032 +炜 4033 +蕙 4034 +鄢 4035 +岑 4036 +恃 4037 +鄱 4038 +拣 4039 +戌 4040 +珲 4041 +窿 4042 +呤 4043 +揪 4044 +羁 4045 +晦 4046 +逞 4047 +摁 4048 +蒡 4049 +殃 4050 +宦 4051 +瞌 4052 +轶 4053 +隅 4054 +掬 4055 +祀 4056 +啉 4057 +噎 4058 +迩 4059 +豇 4060 +汞 4061 +咎 4062 +陟 4063 +毡 4064 +痹 4065 +遨 4066 +铐 4067 +赘 4068 +矗 4069 +砥 4070 +跄 4071 +熨 4072 +漪 4073 +鸾 4074 +绫 4075 +舆 4076 +吠 4077 +犸 4078 +柘 4079 +宸 4080 +陂 4081 +泫 4082 +邸 4083 +萸 4084 +藩 4085 +枰 4086 +撵 4087 +萦 4088 +拴 4089 +绛 4090 +澧 4091 +匮 4092 +轧 4093 +诣 4094 +怅 4095 +盂 4096 +馥 4097 +玥 4098 +垮 4099 +秸 4100 +踉 4101 +荀 4102 +嵘 4103 +暝 4104 +汲 4105 +郯 4106 +耘 4107 +踊 4108 +溯 4109 +瘊 4110 +腼 4111 +町 4112 +倭 4113 +捺 4114 +睬 4115 +踞 4116 +惕 4117 +缔 4118 +篓 4119 +彗 4120 +烀 4121 +辗 4122 +腆 4123 +鲳 4124 +麋 4125 +诃 4126 +沓 4127 +跚 4128 +杳 4129 +姹 4130 +萁 4131 +咫 4132 +稞 4133 +猖 4134 +茏 4135 +竣 4136 +稷 4137 +遛 4138 +厥 4139 +揠 4140 +膺 4141 +煦 4142 +帼 4143 +咻 4144 +挟 4145 +锭 4146 +唔 4147 +敕 4148 +踝 4149 +苋 4150 +麓 4151 +秆 4152 +臊 4153 +蹒 4154 +渎 4155 +赦 4156 +葆 4157 +泞 4158 +驷 4159 +蟠 4160 +诏 4161 +溉 4162 +臧 4163 +濑 4164 +烃 4165 +懵 4166 +藁 4167 +狰 4168 +谧 4169 +湍 4170 +惺 4171 +姊 4172 +捎 4173 +鹜 4174 +敝 4175 +翎 4176 +涎 4177 +勘 4178 +筏 4179 +噶 4180 +痢 4181 +酞 4182 +酣 4183 +缰 4184 +圭 4185 +迥 4186 +潞 4187 +焗 4188 +撬 4189 +邃 4190 +浃 4191 +殆 4192 +嗑 4193 +侬 4194 +骡 4195 +鹄 4196 +酋 4197 +蛆 4198 +偕 4199 +痤 4200 +斟 4201 +豉 4202 +戛 4203 +窘 4204 +碣 4205 +悸 4206 +骋 4207 +趵 4208 +栅 4209 +逑 4210 +钴 4211 +笈 4212 +瞰 4213 +樽 4214 +吆 4215 +榭 4216 +壕 4217 +寮 4218 +拈 4219 +沅 4220 +伫 4221 +扦 4222 +渚 4223 +惦 4224 +舶 4225 +痪 4226 +啧 4227 +褶 4228 +谆 4229 +珈 4230 +庇 4231 +汛 4232 +芩 4233 +婀 4234 +褴 4235 +殉 4236 +佘 4237 +奚 4238 +啾 4239 +荜 4240 +惴 4241 +仟 4242 +褛 4243 +姝 4244 +遏 4245 +搽 4246 +拮 4247 +忾 4248 +奘 4249 +缨 4250 +盱 4251 +洮 4252 +镭 4253 +魇 4254 +蕲 4255 +郦 4256 +蚩 4257 +踹 4258 +泯 4259 +砣 4260 +谬 4261 +昀 4262 +珉 4263 +眙 4264 +贿 4265 +嘈 4266 +褚 4267 +搐 4268 +枷 4269 +硼 4270 +拎 4271 +釉 4272 +鼾 4273 +箔 4274 +帛 4275 +醴 4276 +讹 4277 +惮 4278 +瞅 4279 +孪 4280 +喆 4281 +呋 4282 +郏 4283 +拭 4284 +漉 4285 +糍 4286 +讷 4287 +飚 4288 +瑄 4289 +霁 4290 +咦 4291 +枥 4292 +臼 4293 +祯 4294 +鲛 4295 +蕃 4296 +幢 4297 +笃 4298 +醪 4299 +獭 4300 +邛 4301 +鼬 4302 +瘪 4303 +嗖 4304 +黝 4305 +谙 4306 +嘭 4307 +瘘 4308 +娣 4309 +轱 4310 +瘠 4311 +蠕 4312 +匾 4313 +聩 4314 +歙 4315 +龌 4316 +鳍 4317 +冢 4318 +镕 4319 +荼 4320 +轳 4321 +弋 4322 +胫 4323 +昱 4324 +毋 4325 +缙 4326 +朐 4327 +侥 4328 +栉 4329 +骸 4330 +碉 4331 +讴 4332 +虔 4333 +徇 4334 +濯 4335 +摹 4336 +裱 4337 +庖 4338 +鸪 4339 +怏 4340 +贻 4341 +寰 4342 +叟 4343 +豺 4344 +恪 4345 +暄 4346 +铎 4347 +泱 4348 +浠 4349 +撰 4350 +鬣 4351 +龊 4352 +氰 4353 +札 4354 +蛭 4355 +棣 4356 +痊 4357 +箕 4358 +痂 4359 +哌 4360 +咄 4361 +侗 4362 +蛔 4363 +衅 4364 +妤 4365 +颔 4366 +岫 4367 +炔 4368 +袤 4369 +颚 4370 +宥 4371 +锉 4372 +啶 4373 +偻 4374 +颌 4375 +仃 4376 +扈 4377 +苒 4378 +蹴 4379 +苛 4380 +牍 4381 +骰 4382 +惇 4383 +咿 4384 +伎 4385 +皿 4386 +榈 4387 +缈 4388 +辄 4389 +鹧 4390 +晁 4391 +叩 4392 +崃 4393 +荏 4394 +怆 4395 +蜍 4396 +赉 4397 +滇 4398 +惘 4399 +溥 4400 +儆 4401 +髻 4402 +牦 4403 +沏 4404 +瑗 4405 +蹉 4406 +淞 4407 +渑 4408 +箩 4409 +槛 4410 +蓦 4411 +挝 4412 +楣 4413 +跎 4414 +氩 4415 +铤 4416 +蜊 4417 +芡 4418 +汴 4419 +咛 4420 +绌 4421 +淖 4422 +炀 4423 +祟 4424 +睽 4425 +苜 4426 +灏 4427 +蔫 4428 +菡 4429 +蓿 4430 +孑 4431 +舀 4432 +偎 4433 +坯 4434 +唻 4435 +缥 4436 +怂 4437 +曛 4438 +褥 4439 +毂 4440 +瞑 4441 +戍 4442 +玷 4443 +撮 4444 +莜 4445 +讪 4446 +赡 4447 +鹩 4448 +徨 4449 +罡 4450 +荸 4451 +皙 4452 +跺 4453 +彷 4454 +噔 4455 +荃 4456 +灞 4457 +狒 4458 +啕 4459 +赳 4460 +焱 4461 +烬 4462 +拗 4463 +篡 4464 +袱 4465 +濂 4466 +弼 4467 +唬 4468 +赃 4469 +囱 4470 +滂 4471 +鄄 4472 +惆 4473 +谤 4474 +搀 4475 +阐 4476 +嚓 4477 +圩 4478 +蹂 4479 +骜 4480 +铄 4481 +膻 4482 +藐 4483 +畴 4484 +磋 4485 +獾 4486 +甯 4487 +邈 4488 +佝 4489 +禀 4490 +嗤 4491 +桀 4492 +钎 4493 +羟 4494 +醺 4495 +卞 4496 +酐 4497 +颧 4498 +吡 4499 +狍 4500 +酉 4501 +垛 4502 +涝 4503 +臾 4504 +胍 4505 +锚 4506 +蜚 4507 +貉 4508 +愕 4509 +锢 4510 +酊 4511 +蜉 4512 +迤 4513 +铣 4514 +揖 4515 +钨 4516 +覃 4517 +晌 4518 +掣 4519 +凇 4520 +匝 4521 +栎 4522 +阉 4523 +汨 4524 +蚬 4525 +俨 4526 +觞 4527 +鳜 4528 +恻 4529 +钣 4530 +谕 4531 +醚 4532 +仨 4533 +铬 4534 +刈 4535 +衿 4536 +贰 4537 +窒 4538 +丕 4539 +犟 4540 +躏 4541 +汆 4542 +饴 4543 +厮 4544 +簌 4545 +饕 4546 +嗣 4547 +扼 4548 +儋 4549 +睑 4550 +驽 4551 +堑 4552 +娑 4553 +隼 4554 +濠 4555 +忪 4556 +噻 4557 +漕 4558 +茕 4559 +缄 4560 +锹 4561 +俎 4562 +螈 4563 +亘 4564 +镍 4565 +瀛 4566 +绦 4567 +岌 4568 +戾 4569 +胛 4570 +翊 4571 +篑 4572 +芎 4573 +銮 4574 +嘌 4575 +毖 4576 +掂 4577 +贮 4578 +幡 4579 +煊 4580 +邡 4581 +劵 4582 +濉 4583 +髦 4584 +茌 4585 +囤 4586 +夙 4587 +蜢 4588 +诠 4589 +黍 4590 +恣 4591 +诬 4592 +氦 4593 +舐 4594 +睐 4595 +谗 4596 +嘣 4597 +肟 4598 +鲲 4599 +塾 4600 +肏 4601 +砾 4602 +罄 4603 +苁 4604 +艹 4605 +玑 4606 +旌 4607 +鲽 4608 +搪 4609 +捍 4610 +逶 4611 +阖 4612 +煽 4613 +殚 4614 +伥 4615 +耷 4616 +叁 4617 +枞 4618 +嗟 4619 +芨 4620 +辍 4621 +踮 4622 +蛳 4623 +箬 4624 +屐 4625 +娌 4626 +箴 4627 +蝾 4628 +柒 4629 +琨 4630 +犄 4631 +蚜 4632 +膑 4633 +涔 4634 +槌 4635 +裟 4636 +萘 4637 +菟 4638 +嬴 4639 +琍 4640 +忱 4641 +靳 4642 +髋 4643 +旮 4644 +抻 4645 +餮 4646 +郧 4647 +飕 4648 +涠 4649 +煅 4650 +饽 4651 +眦 4652 +噤 4653 +茭 4654 +迄 4655 +苄 4656 +箪 4657 +诽 4658 +掸 4659 +谀 4660 +悖 4661 +瓤 4662 +聿 4663 +憩 4664 +谩 4665 +疖 4666 +藜 4667 +阚 4668 +蛏 4669 +壬 4670 +肱 4671 +妯 4672 +篙 4673 +蜱 4674 +垩 4675 +樨 4676 +悌 4677 +囔 4678 +曈 4679 +脲 4680 +啫 4681 +彧 4682 +碴 4683 +赊 4684 +篝 4685 +鳃 4686 +霭 4687 +呷 4688 +焙 4689 +溏 4690 +萼 4691 +徉 4692 +倨 4693 +砒 4694 +坳 4695 +桧 4696 +颢 4697 +拄 4698 +颉 4699 +砭 4700 +镳 4701 +桅 4702 +伢 4703 +燮 4704 +缛 4705 +裨 4706 +粕 4707 +臃 4708 +魍 4709 +蝣 4710 +荇 4711 +疥 4712 +唛 4713 +愠 4714 +麸 4715 +蜷 4716 +徜 4717 +魉 4718 +耦 4719 +樾 4720 +瓴 4721 +竦 4722 +扪 4723 +嵬 4724 +袈 4725 +谡 4726 +秣 4727 +晷 4728 +浥 4729 +宕 4730 +哧 4731 +攥 4732 +瞥 4733 +坍 4734 +怔 4735 +犷 4736 +摞 4737 +撷 4738 +熘 4739 +巽 4740 +湟 4741 +倜 4742 +鳊 4743 +鬃 4744 +磬 4745 +傥 4746 +菈 4747 +捻 4748 +畦 4749 +翳 4750 +侑 4751 +厝 4752 +洵 4753 +瞠 4754 +臆 4755 +忒 4756 +跬 4757 +璜 4758 +醍 4759 +吮 4760 +蒌 4761 +邺 4762 +僚 4763 +虬 4764 +蹿 4765 +綦 4766 +羧 4767 +诘 4768 +姣 4769 +楹 4770 +忸 4771 +铆 4772 +邬 4773 +踱 4774 +隘 4775 +怩 4776 +滢 4777 +喑 4778 +魑 4779 +龇 4780 +冼 4781 +哽 4782 +譬 4783 +迸 4784 +仄 4785 +赅 4786 +邰 4787 +淙 4788 +旯 4789 +莠 4790 +蹩 4791 +晤 4792 +诙 4793 +谟 4794 +佞 4795 +雉 4796 +砗 4797 +磲 4798 +誊 4799 +钒 4800 +膘 4801 +旖 4802 +匍 4803 +慑 4804 +恹 4805 +暹 4806 +踽 4807 +馕 4808 +晔 4809 +淆 4810 +崆 4811 +刎 4812 +纭 4813 +罘 4814 +桉 4815 +杈 4816 +淝 4817 +菅 4818 +鳐 4819 +谪 4820 +肓 4821 +盹 4822 +殒 4823 +旎 4824 +掼 4825 +柞 4826 +鲮 4827 +癀 4828 +岈 4829 +黜 4830 +铳 4831 +甬 4832 +匐 4833 +嗔 4834 +琰 4835 +犒 4836 +绉 4837 +闾 4838 +醐 4839 +蓥 4840 +媲 4841 +陉 4842 +蹶 4843 +髫 4844 +篁 4845 +勐 4846 +钵 4847 +笤 4848 +猗 4849 +皈 4850 +埂 4851 +泮 4852 +哓 4853 +腭 4854 +窠 4855 +隰 4856 +悱 4857 +芘 4858 +铰 4859 +芾 4860 +舫 4861 +摒 4862 +胄 4863 +嗲 4864 +骶 4865 +湮 4866 +诨 4867 +巳 4868 +靛 4869 +崮 4870 +盅 4871 +唆 4872 +鸬 4873 +菖 4874 +袒 4875 +鲑 4876 +砝 4877 +佚 4878 +倏 4879 +珩 4880 +痍 4881 +孬 4882 +绾 4883 +宓 4884 +坼 4885 +聒 4886 +赂 4887 +峒 4888 +擢 4889 +趄 4890 +啜 4891 +橼 4892 +衮 4893 +帧 4894 +螯 4895 +鹚 4896 +枳 4897 +嘤 4898 +愎 4899 +幔 4900 +荪 4901 +遽 4902 +馊 4903 +袂 4904 +鳟 4905 +瓠 4906 +贲 4907 +窸 4908 +挈 4909 +捋 4910 +沆 4911 +哞 4912 +椹 4913 +蓖 4914 +睚 4915 +渥 4916 +杼 4917 +棹 4918 +剽 4919 +屙 4920 +鸩 4921 +薹 4922 +珙 4923 +邙 4924 +瑁 4925 +铋 4926 +樯 4927 +滓 4928 +埭 4929 +碚 4930 +慵 4931 +膈 4932 +砧 4933 +毗 4934 +夯 4935 +徕 4936 +粳 4937 +橹 4938 +窣 4939 +獠 4940 +镊 4941 +箸 4942 +蘼 4943 +嬅 4944 +趔 4945 +吲 4946 +珅 4947 +喙 4948 +獐 4949 +璎 4950 +蛄 4951 +髌 4952 +珏 4953 +楫 4954 +倬 4955 +颏 4956 +骠 4957 +骝 4958 +悛 4959 +垭 4960 +赓 4961 +邝 4962 +秭 4963 +脘 4964 +糸 4965 +拏 4966 +堇 4967 +蛱 4968 +鎏 4969 +祎 4970 +谒 4971 +埚 4972 +怙 4973 +溆 4974 +黟 4975 +的 4976 +a 4977 +b 4978 +c 4979 +d 4980 +e 4981 +f 4982 +g 4983 +h 4984 +i 4985 +j 4986 +k 4987 +l 4988 +m 4989 +n 4990 +o 4991 +p 4992 +q 4993 +r 4994 +s 4995 +t 4996 +u 4997 +v 4998 +w 4999 +x 5000 +y 5001 +z 5002 +- 5003 + 5004 diff --git a/athena/utils/vocabs/ch.vocab b/athena/utils/vocabs/ch.vocab new file mode 100644 index 00000000..4f353354 --- /dev/null +++ b/athena/utils/vocabs/ch.vocab @@ -0,0 +1,4977 @@ + 0 +么 1 +什 2 +一 3 +是 4 +小 5 +大 6 +怎 7 +有 8 +人 9 +片 10 +天 11 +二 12 +我 13 +电 14 +子 15 +三 16 +图 17 +语 18 +女 19 +词 20 +下 21 +视 22 +年 23 +之 24 +生 25 +游 26 +十 27 +手 28 +文 29 +五 30 +中 31 +多 32 +英 33 +不 34 +四 35 +戏 36 +国 37 +哪 38 +字 39 +吗 40 +动 41 +机 42 +上 43 +个 44 +学 45 +美 46 +车 47 +少 48 +好 49 +影 50 +第 51 +画 52 +歌 53 +频 54 +零 55 +新 56 +成 57 +意 58 +在 59 +儿 60 +爱 61 +看 62 +宝 63 +载 64 +用 65 +版 66 +作 67 +思 68 +义 69 +全 70 +战 71 +公 72 +说 73 +花 74 +头 75 +最 76 +六 77 +世 78 +和 79 +写 80 +百 81 +神 82 +要 83 +主 84 +集 85 +王 86 +里 87 +能 88 +于 89 +样 90 +级 91 +地 92 +了 93 +做 94 +特 95 +出 96 +山 97 +气 98 +物 99 +春 100 +界 101 +家 102 +以 103 +可 104 +日 105 +水 106 +乐 107 +西 108 +关 109 +名 110 +星 111 +长 112 +事 113 +时 114 +海 115 +金 116 +色 117 +法 118 +心 119 +九 120 +奇 121 +打 122 +句 123 +八 124 +龙 125 +开 126 +风 127 +吃 128 +老 129 +七 130 +曲 131 +你 132 +到 133 +近 134 +会 135 +高 136 +情 137 +火 138 +方 139 +熊 140 +没 141 +剧 142 +后 143 +为 144 +来 145 +音 146 +羊 147 +月 148 +号 149 +记 150 +明 151 +照 152 +果 153 +爸 154 +网 155 +奥 156 +安 157 +报 158 +跑 159 +市 160 +节 161 +南 162 +发 163 +话 164 +东 165 +男 166 +组 167 +面 168 +魔 169 +黄 170 +点 171 +马 172 +梦 173 +鱼 174 +读 175 +红 176 +州 177 +白 178 +反 179 +猪 180 +舞 181 +数 182 +谁 183 +妈 184 +传 185 +如 186 +士 187 +城 188 +分 189 +侠 190 +些 191 +曼 192 +拉 193 +千 194 +快 195 +解 196 +过 197 +几 198 +阳 199 +光 200 +飞 201 +行 202 +无 203 +比 204 +单 205 +玩 206 +线 207 +想 208 +部 209 +场 210 +喜 211 +自 212 +加 213 +季 214 +巴 215 +钱 216 +化 217 +诗 218 +书 219 +感 220 +尸 221 +仙 222 +性 223 +本 224 +体 225 +与 226 +北 227 +信 228 +孩 229 +江 230 +广 231 +酷 232 +雪 233 +演 234 +古 235 +去 236 +平 237 +吧 238 +甲 239 +赛 240 +僵 241 +道 242 +间 243 +形 244 +度 245 +表 246 +故 247 +期 248 +米 249 +对 250 +园 251 +超 252 +变 253 +格 254 +活 255 +啦 256 +李 257 +张 258 +器 259 +河 260 +工 261 +何 262 +香 263 +都 264 +林 265 +空 266 +品 267 +联 268 +丽 269 +题 270 +尔 271 +者 272 +雄 273 +路 274 +代 275 +太 276 +衣 277 +民 278 +放 279 +笔 280 +漫 281 +罗 282 +童 283 +师 284 +青 285 +册 286 +结 287 +克 288 +鸡 289 +斯 290 +华 291 +门 292 +相 293 +精 294 +冰 295 +弟 296 +预 297 +身 298 +球 299 +真 300 +笑 301 +播 302 +通 303 +猫 304 +卡 305 +简 306 +服 307 +等 308 +合 309 +业 310 +重 311 +理 312 +还 313 +京 314 +口 315 +两 316 +石 317 +装 318 +强 319 +狗 320 +回 321 +黑 322 +元 323 +起 324 +植 325 +像 326 +晚 327 +院 328 +苹 329 +万 330 +流 331 +穿 332 +清 333 +教 334 +经 335 +福 336 +包 337 +价 338 +兄 339 +雨 340 +区 341 +现 342 +办 343 +内 344 +微 345 +原 346 +武 347 +式 348 +亚 349 +斗 350 +假 351 +码 352 +铁 353 +带 354 +声 355 +蛋 356 +前 357 +勇 358 +案 359 +言 360 +费 361 +灵 362 +这 363 +铠 364 +今 365 +保 366 +种 367 +破 368 +叫 369 +容 370 +击 371 +豆 372 +云 373 +力 374 +娃 375 +欢 376 +医 377 +正 378 +外 379 +唱 380 +同 381 +死 382 +造 383 +贝 384 +见 385 +票 386 +房 387 +友 388 +别 389 +彩 390 +得 391 +杨 392 +越 393 +答 394 +队 395 +草 396 +皮 397 +奔 398 +所 399 +台 400 +查 401 +险 402 +考 403 +陈 404 +牛 405 +交 406 +校 407 +周 408 +狂 409 +连 410 +习 411 +岁 412 +那 413 +们 414 +件 415 +亲 416 +双 417 +翻 418 +搜 419 +制 420 +科 421 +县 422 +叶 423 +座 424 +拼 425 +候 426 +湖 427 +奶 428 +眼 429 +血 430 +刘 431 +知 432 +具 433 +历 434 +观 435 +卫 436 +利 437 +密 438 +菜 439 +雷 440 +汽 441 +酒 442 +夜 443 +店 444 +员 445 +迪 446 +应 447 +银 448 +试 449 +木 450 +给 451 +听 452 +才 453 +肉 454 +免 455 +芭 456 +苏 457 +树 458 +运 459 +德 460 +艺 461 +目 462 +治 463 +指 464 +景 465 +毛 466 +抄 467 +当 468 +消 469 +股 470 +着 471 +狼 472 +纸 473 +译 474 +韩 475 +修 476 +婚 477 +药 478 +油 479 +被 480 +完 481 +站 482 +娘 483 +达 484 +省 485 +计 486 +干 487 +常 488 +食 489 +兵 490 +姐 491 +怪 492 +描 493 +程 494 +料 495 +啊 496 +先 497 +阿 498 +旁 499 +找 500 +然 501 +首 502 +汉 503 +鬼 504 +虎 505 +产 506 +易 507 +孕 508 +洋 509 +屁 510 +骨 511 +走 512 +请 513 +角 514 +兽 515 +直 516 +哥 517 +枪 518 +位 519 +索 520 +阅 521 +术 522 +极 523 +实 524 +进 525 +资 526 +总 527 +兰 528 +母 529 +他 530 +定 531 +处 532 +妇 533 +宁 534 +吉 535 +洗 536 +系 537 +除 538 +因 539 +初 540 +亮 541 +泰 542 +唐 543 +疯 544 +基 545 +剑 546 +村 547 +边 548 +排 549 +恐 550 +功 551 +志 552 +丝 553 +夫 554 +寒 555 +很 556 +速 557 +步 558 +兔 559 +赵 560 +招 561 +脑 562 +灰 563 +刚 564 +热 565 +列 566 +酸 567 +养 568 +算 569 +朋 570 +闻 571 +皇 572 +鸟 573 +军 574 +圣 575 +病 576 +司 577 +入 578 +盗 579 +妹 580 +郑 581 +属 582 +秘 583 +岛 584 +脱 585 +玉 586 +模 587 +问 588 +烧 589 +限 590 +冒 591 +沙 592 +型 593 +只 594 +蓝 595 +牌 596 +哈 597 +旋 598 +夏 599 +幻 600 +就 601 +农 602 +量 603 +博 604 +商 605 +建 606 +鼠 607 +汤 608 +深 609 +示 610 +裸 611 +短 612 +使 613 +警 614 +官 615 +从 616 +买 617 +喝 618 +转 619 +乡 620 +课 621 +妖 622 +野 623 +粉 624 +远 625 +偏 626 +立 627 +莉 628 +阴 629 +软 630 +秋 631 +录 632 +类 633 +土 634 +恶 635 +底 636 +次 637 +温 638 +糖 639 +菲 640 +调 641 +卖 642 +巨 643 +盟 644 +离 645 +源 646 +板 647 +拍 648 +素 649 +珠 650 +萝 651 +床 652 +管 653 +客 654 +萌 655 +接 656 +把 657 +复 658 +灯 659 +封 660 +布 661 +鞋 662 +朵 663 +护 664 +搞 665 +配 666 +优 667 +怀 668 +改 669 +营 670 +跳 671 +孙 672 +兴 673 +非 674 +将 675 +族 676 +局 677 +炎 678 +宫 679 +而 680 +婆 681 +桃 682 +命 683 +馆 684 +豪 685 +助 686 +向 687 +设 688 +操 689 +范 690 +恋 691 +技 692 +维 693 +章 694 +奖 695 +雅 696 +史 697 +凯 698 +杀 699 +换 700 +圆 701 +耳 702 +杰 703 +田 704 +整 705 +镇 706 +峰 707 +收 708 +摩 709 +逼 710 +减 711 +泡 712 +标 713 +己 714 +态 715 +虫 716 +贵 717 +缘 718 +育 719 +波 720 +环 721 +含 722 +楼 723 +朝 724 +饭 725 +落 726 +根 727 +智 728 +洛 729 +存 730 +争 731 +团 732 +托 733 +绝 734 +壁 735 +专 736 +条 737 +猎 738 +难 739 +称 740 +质 741 +介 742 +币 743 +告 744 +绿 745 +牙 746 +宠 747 +积 748 +象 749 +睡 750 +班 751 +呢 752 +欧 753 +佳 754 +息 755 +颜 756 +尼 757 +帮 758 +帝 759 +秀 760 +岭 761 +妻 762 +颖 763 +求 764 +瓜 765 +望 766 +呀 767 +拿 768 +妙 769 +置 770 +乘 771 +烟 772 +暖 773 +探 774 +激 775 +半 776 +折 777 +蹈 778 +梅 779 +川 780 +芦 781 +跟 782 +晓 783 +冬 784 +盒 785 +裤 786 +典 787 +乳 788 +旅 789 +盘 790 +港 791 +亡 792 +幸 793 +庆 794 +刀 795 +便 796 +麻 797 +证 798 +爆 799 +取 800 +肥 801 +钢 802 +威 803 +妃 804 +异 805 +漂 806 +凤 807 +暴 808 +麦 809 +防 810 +胸 811 +编 812 +提 813 +逃 814 +升 815 +背 816 +统 817 +茶 818 +雕 819 +庄 820 +康 821 +坐 822 +脚 823 +猜 824 +健 825 +丁 826 +济 827 +塔 828 +又 829 +需 830 +让 831 +腾 832 +鲁 833 +该 834 +吻 835 +陆 836 +吴 837 +至 838 +更 839 +纪 840 +蛇 841 +贼 842 +务 843 +讲 844 +榜 845 +卷 846 +侣 847 +钟 848 +其 849 +驾 850 +紫 851 +也 852 +佛 853 +俊 854 +共 855 +痛 856 +礼 857 +瑞 858 +政 859 +骑 860 +忍 861 +乌 862 +脸 863 +未 864 +洲 865 +充 866 +绵 867 +伤 868 +毒 869 +顺 870 +糕 871 +约 872 +霸 873 +练 874 +父 875 +续 876 +觉 877 +导 878 +邮 879 +迹 880 +社 881 +室 882 +篇 883 +攻 884 +套 885 +款 886 +再 887 +早 888 +登 889 +满 890 +堂 891 +葫 892 +寓 893 +松 894 +静 895 +值 896 +由 897 +奸 898 +谜 899 +甜 900 +状 901 +帅 902 +露 903 +效 904 +冲 905 +贴 906 +识 907 +冷 908 +街 909 +嘴 910 +偷 911 +询 912 +送 913 +桥 914 +普 915 +胡 916 +藏 917 +段 918 +幼 919 +浪 920 +临 921 +屏 922 +逆 923 +炒 924 +丹 925 +氧 926 +归 927 +祝 928 +嗯 929 +狙 930 +足 931 +蜜 932 +丰 933 +永 934 +注 935 +失 936 +富 937 +弹 938 +瓦 939 +袋 940 +宋 941 +蜡 942 +娜 943 +谢 944 +妆 945 +测 946 +念 947 +恩 948 +射 949 +守 950 +左 951 +姆 952 +航 953 +箱 954 +久 955 +末 956 +尾 957 +锋 958 +君 959 +乱 960 +啥 961 +选 962 +疼 963 +爷 964 +蒙 965 +宇 966 +卜 967 +始 968 +迦 969 +燕 970 +犬 971 +附 972 +宵 973 +迷 974 +巧 975 +份 976 +伦 977 +轻 978 +味 979 +住 980 +齐 981 +购 982 +琴 983 +疗 984 +滨 985 +邪 986 +莲 987 +压 988 +荣 989 +论 990 +症 991 +姑 992 +终 993 +补 994 +谱 995 +受 996 +昌 997 +轮 998 +佩 999 +各 1000 +居 1001 +尿 1002 +魂 1003 +掉 1004 +镜 1005 +鹿 1006 +徐 1007 +独 1008 +胜 1009 +讯 1010 +倒 1011 +绍 1012 +肚 1013 +锁 1014 +已 1015 +央 1016 +决 1017 +滑 1018 +块 1019 +害 1020 +势 1021 +群 1022 +际 1023 +泉 1024 +输 1025 +怒 1026 +趣 1027 +右 1028 +及 1029 +屋 1030 +材 1031 +敢 1032 +姓 1033 +液 1034 +厂 1035 +划 1036 +拳 1037 +创 1038 +征 1039 +沈 1040 +腿 1041 +突 1042 +晨 1043 +则 1044 +惊 1045 +竹 1046 +验 1047 +泽 1048 +毕 1049 +辣 1050 +悟 1051 +憨 1052 +歇 1053 +细 1054 +捕 1055 +凡 1056 +任 1057 +律 1058 +遇 1059 +狐 1060 +柴 1061 +吸 1062 +支 1063 +财 1064 +拟 1065 +柳 1066 +祖 1067 +暗 1068 +猴 1069 +聘 1070 +邓 1071 +略 1072 +职 1073 +每 1074 +嘉 1075 +津 1076 +滴 1077 +秦 1078 +若 1079 +玛 1080 +参 1081 +淘 1082 +庙 1083 +裙 1084 +良 1085 +珍 1086 +铃 1087 +敌 1088 +适 1089 +尽 1090 +呼 1091 +莫 1092 +插 1093 +鸭 1094 +锦 1095 +针 1096 +瓶 1097 +规 1098 +货 1099 +挂 1100 +炼 1101 +肖 1102 +依 1103 +苦 1104 +胶 1105 +忘 1106 +盛 1107 +释 1108 +宜 1109 +晶 1110 +谷 1111 +坏 1112 +引 1113 +架 1114 +错 1115 +络 1116 +裁 1117 +庭 1118 +炮 1119 +切 1120 +龟 1121 +莱 1122 +堡 1123 +戴 1124 +移 1125 +欲 1126 +刷 1127 +偶 1128 +摇 1129 +顶 1130 +评 1131 +炸 1132 +霆 1133 +艾 1134 +泥 1135 +蝶 1136 +急 1137 +鼻 1138 +递 1139 +凉 1140 +午 1141 +澡 1142 +领 1143 +致 1144 +朱 1145 +柯 1146 +权 1147 +众 1148 +餐 1149 +苍 1150 +降 1151 +婴 1152 +棋 1153 +某 1154 +船 1155 +伟 1156 +透 1157 +丸 1158 +腹 1159 +许 1160 +备 1161 +钻 1162 +察 1163 +印 1164 +盐 1165 +伊 1166 +低 1167 +持 1168 +诺 1169 +闹 1170 +户 1171 +遥 1172 +鲨 1173 +俗 1174 +戒 1175 +杜 1176 +纳 1177 +郎 1178 +浩 1179 +令 1180 +孔 1181 +况 1182 +杭 1183 +必 1184 +危 1185 +森 1186 +傻 1187 +寻 1188 +盖 1189 +膜 1190 +尊 1191 +控 1192 +睛 1193 +付 1194 +蛛 1195 +幂 1196 +艳 1197 +腰 1198 +派 1199 +刻 1200 +举 1201 +刑 1202 +括 1203 +敏 1204 +较 1205 +哆 1206 +鹅 1207 +杯 1208 +夺 1209 +蜂 1210 +祥 1211 +臭 1212 +断 1213 +玫 1214 +境 1215 +尚 1216 +梁 1217 +虹 1218 +郭 1219 +赞 1220 +胃 1221 +鲜 1222 +蜘 1223 +闪 1224 +签 1225 +展 1226 +悠 1227 +眉 1228 +饼 1229 +启 1230 +默 1231 +旗 1232 +湾 1233 +锅 1234 +聊 1235 +仁 1236 +侦 1237 +救 1238 +响 1239 +桂 1240 +萨 1241 +似 1242 +慧 1243 +肤 1244 +瑰 1245 +怖 1246 +填 1247 +棒 1248 +圳 1249 +待 1250 +扬 1251 +推 1252 +辽 1253 +愤 1254 +丘 1255 +燃 1256 +希 1257 +荷 1258 +徽 1259 +桌 1260 +摸 1261 +灭 1262 +址 1263 +冠 1264 +端 1265 +诚 1266 +池 1267 +籍 1268 +隐 1269 +围 1270 +溪 1271 +伴 1272 +核 1273 +杉 1274 +炫 1275 +卓 1276 +胎 1277 +煮 1278 +项 1279 +宾 1280 +狮 1281 +益 1282 +馀 1283 +塞 1284 +扫 1285 +序 1286 +止 1287 +潮 1288 +负 1289 +惠 1290 +途 1291 +樱 1292 +善 1293 +拜 1294 +凰 1295 +囊 1296 +伯 1297 +昆 1298 +咪 1299 +洞 1300 +赤 1301 +旧 1302 +熟 1303 +蒸 1304 +匙 1305 +往 1306 +准 1307 +欣 1308 +劳 1309 +腐 1310 +退 1311 +钥 1312 +愿 1313 +虚 1314 +虾 1315 +霉 1316 +批 1317 +蝴 1318 +烈 1319 +禁 1320 +饰 1321 +朗 1322 +增 1323 +横 1324 +申 1325 +追 1326 +按 1327 +捉 1328 +随 1329 +浙 1330 +般 1331 +钓 1332 +挖 1333 +胆 1334 +仪 1335 +租 1336 +舒 1337 +检 1338 +慢 1339 +闯 1340 +媚 1341 +肠 1342 +休 1343 +坦 1344 +账 1345 +拔 1346 +综 1347 +谭 1348 +沉 1349 +曹 1350 +夕 1351 +率 1352 +墙 1353 +隆 1354 +鹰 1355 +圈 1356 +厅 1357 +弄 1358 +叔 1359 +震 1360 +泳 1361 +贺 1362 +扎 1363 +刺 1364 +玲 1365 +剪 1366 +蔬 1367 +罪 1368 +恺 1369 +袜 1370 +舍 1371 +洁 1372 +叉 1373 +吹 1374 +概 1375 +绣 1376 +井 1377 +倍 1378 +湿 1379 +留 1380 +喻 1381 +投 1382 +浒 1383 +炖 1384 +净 1385 +耀 1386 +奏 1387 +染 1388 +怕 1389 +抗 1390 +梨 1391 +雀 1392 +掌 1393 +芳 1394 +凌 1395 +采 1396 +篮 1397 +碧 1398 +鼓 1399 +菇 1400 +爽 1401 +否 1402 +桶 1403 +谈 1404 +汇 1405 +纯 1406 +洪 1407 +孤 1408 +扮 1409 +墓 1410 +碳 1411 +寺 1412 +羽 1413 +仔 1414 +铜 1415 +销 1416 +构 1417 +废 1418 +例 1419 +散 1420 +键 1421 +氢 1422 +苗 1423 +认 1424 +岳 1425 +顾 1426 +傲 1427 +唯 1428 +饺 1429 +抱 1430 +陵 1431 +雾 1432 +抽 1433 +斑 1434 +库 1435 +荒 1436 +魅 1437 +鹤 1438 +售 1439 +承 1440 +忆 1441 +琪 1442 +舟 1443 +敬 1444 +痘 1445 +败 1446 +瘦 1447 +龄 1448 +胖 1449 +甘 1450 +叠 1451 +聚 1452 +训 1453 +删 1454 +笨 1455 +滩 1456 +澳 1457 +显 1458 +窗 1459 +倾 1460 +幽 1461 +厘 1462 +惑 1463 +挑 1464 +泪 1465 +螺 1466 +严 1467 +狱 1468 +亦 1469 +烤 1470 +晗 1471 +晴 1472 +驶 1473 +截 1474 +企 1475 +获 1476 +符 1477 +碰 1478 +丑 1479 +据 1480 +抢 1481 +醉 1482 +驰 1483 +私 1484 +停 1485 +席 1486 +此 1487 +映 1488 +迅 1489 +芝 1490 +翔 1491 +翼 1492 +纹 1493 +汪 1494 +她 1495 +贤 1496 +嫁 1497 +娇 1498 +氏 1499 +殖 1500 +悦 1501 +帽 1502 +壮 1503 +枝 1504 +饿 1505 +厉 1506 +饮 1507 +笼 1508 +寿 1509 +混 1510 +溜 1511 +唤 1512 +钠 1513 +醋 1514 +府 1515 +织 1516 +肝 1517 +赏 1518 +硫 1519 +宅 1520 +牧 1521 +厦 1522 +岩 1523 +焰 1524 +裂 1525 +霜 1526 +确 1527 +幕 1528 +召 1529 +翅 1530 +颗 1531 +润 1532 +蟹 1533 +涛 1534 +咋 1535 +辉 1536 +姜 1537 +差 1538 +婷 1539 +扣 1540 +仿 1541 +咳 1542 +斤 1543 +坚 1544 +抓 1545 +箭 1546 +赫 1547 +繁 1548 +宏 1549 +曾 1550 +淮 1551 +诱 1552 +厨 1553 +鳄 1554 +副 1555 +肌 1556 +葡 1557 +贾 1558 +菊 1559 +玺 1560 +硬 1561 +尺 1562 +墨 1563 +莎 1564 +轩 1565 +肿 1566 +览 1567 +潘 1568 +仓 1569 +萄 1570 +爹 1571 +借 1572 +即 1573 +搭 1574 +吐 1575 +页 1576 +哭 1577 +匆 1578 +乔 1579 +撸 1580 +乙 1581 +弯 1582 +沟 1583 +蛙 1584 +委 1585 +尖 1586 +塘 1587 +继 1588 +喂 1589 +悲 1590 +孝 1591 +暑 1592 +茄 1593 +纵 1594 +梯 1595 +绘 1596 +薇 1597 +宽 1598 +坛 1599 +勤 1600 +楚 1601 +坊 1602 +舌 1603 +穷 1604 +卧 1605 +鸣 1606 +乖 1607 +闲 1608 +饥 1609 +溶 1610 +醒 1611 +痒 1612 +宗 1613 +仇 1614 +咭 1615 +柔 1616 +施 1617 +亭 1618 +毫 1619 +豫 1620 +烊 1621 +飘 1622 +迎 1623 +议 1624 +酱 1625 +违 1626 +宣 1627 +淫 1628 +层 1629 +滚 1630 +辛 1631 +固 1632 +嫣 1633 +纱 1634 +膏 1635 +椒 1636 +姿 1637 +柏 1638 +宿 1639 +恒 1640 +柱 1641 +吊 1642 +捷 1643 +腺 1644 +孟 1645 +胞 1646 +徒 1647 +粒 1648 +或 1649 +哦 1650 +肾 1651 +诉 1652 +丫 1653 +霞 1654 +颈 1655 +萧 1656 +霍 1657 +汗 1658 +绩 1659 +残 1660 +豹 1661 +烂 1662 +枣 1663 +浦 1664 +桑 1665 +姨 1666 +蹦 1667 +齿 1668 +呃 1669 +尘 1670 +淋 1671 +薄 1672 +扇 1673 +嘟 1674 +棉 1675 +汁 1676 +辑 1677 +稀 1678 +培 1679 +疾 1680 +懒 1681 +漠 1682 +惜 1683 +著 1684 +锡 1685 +杂 1686 +玄 1687 +踩 1688 +咸 1689 +拌 1690 +革 1691 +莓 1692 +靠 1693 +菠 1694 +陀 1695 +峡 1696 +眠 1697 +绑 1698 +陕 1699 +撕 1700 +陶 1701 +菱 1702 +擦 1703 +葱 1704 +筋 1705 +并 1706 +盆 1707 +紧 1708 +哒 1709 +述 1710 +逗 1711 +档 1712 +媳 1713 +伏 1714 +忧 1715 +延 1716 +璃 1717 +辰 1718 +析 1719 +寂 1720 +涵 1721 +橡 1722 +荡 1723 +顿 1724 +域 1725 +缺 1726 +氯 1727 +屎 1728 +邻 1729 +遗 1730 +黎 1731 +翠 1732 +冻 1733 +菌 1734 +励 1735 +衡 1736 +穴 1737 +奋 1738 +涯 1739 +监 1740 +咖 1741 +狸 1742 +磨 1743 +董 1744 +疆 1745 +姚 1746 +玻 1747 +番 1748 +弃 1749 +哺 1750 +拖 1751 +赶 1752 +究 1753 +猛 1754 +摄 1755 +妮 1756 +芒 1757 +脏 1758 +咬 1759 +鸿 1760 +粘 1761 +它 1762 +研 1763 +聪 1764 +蕉 1765 +疹 1766 +薛 1767 +壳 1768 +鹏 1769 +订 1770 +摆 1771 +坑 1772 +彼 1773 +喷 1774 +逊 1775 +诞 1776 +卵 1777 +链 1778 +蔡 1779 +炉 1780 +彭 1781 +钙 1782 +杏 1783 +督 1784 +额 1785 +鞭 1786 +淡 1787 +葛 1788 +浮 1789 +膀 1790 +怡 1791 +涂 1792 +牡 1793 +焦 1794 +驴 1795 +鼎 1796 +距 1797 +累 1798 +浴 1799 +俏 1800 +稿 1801 +缩 1802 +割 1803 +腔 1804 +宙 1805 +忠 1806 +党 1807 +筒 1808 +橙 1809 +馨 1810 +须 1811 +芽 1812 +晋 1813 +竞 1814 +琳 1815 +娱 1816 +骚 1817 +嗽 1818 +避 1819 +筝 1820 +懂 1821 +寸 1822 +蒂 1823 +穹 1824 +摘 1825 +乃 1826 +萍 1827 +蝎 1828 +互 1829 +诸 1830 +御 1831 +窝 1832 +巾 1833 +瓷 1834 +坡 1835 +阜 1836 +魏 1837 +隋 1838 +腊 1839 +矿 1840 +肯 1841 +旺 1842 +扑 1843 +厚 1844 +判 1845 +闭 1846 +莹 1847 +韵 1848 +迁 1849 +阵 1850 +烦 1851 +惹 1852 +癌 1853 +嘛 1854 +浏 1855 +脂 1856 +蒲 1857 +茅 1858 +柜 1859 +貌 1860 +妍 1861 +啪 1862 +绳 1863 +蒜 1864 +沧 1865 +弱 1866 +筑 1867 +疑 1868 +犯 1869 +磁 1870 +淇 1871 +执 1872 +陌 1873 +泊 1874 +逸 1875 +垃 1876 +误 1877 +竖 1878 +侧 1879 +渡 1880 +潜 1881 +辞 1882 +芬 1883 +弊 1884 +忙 1885 +罐 1886 +碎 1887 +罩 1888 +熙 1889 +圾 1890 +袭 1891 +困 1892 +叮 1893 +俄 1894 +寄 1895 +勒 1896 +荆 1897 +戚 1898 +岸 1899 +铺 1900 +束 1901 +妞 1902 +融 1903 +咏 1904 +剂 1905 +朴 1906 +撒 1907 +甄 1908 +谚 1909 +亿 1910 +辆 1911 +杆 1912 +蓉 1913 +庐 1914 +浆 1915 +鸽 1916 +蒋 1917 +粗 1918 +耶 1919 +添 1920 +翁 1921 +蚂 1922 +垂 1923 +踪 1924 +荐 1925 +辨 1926 +蘑 1927 +蚁 1928 +漆 1929 +笙 1930 +策 1931 +慰 1932 +沂 1933 +但 1934 +柿 1935 +潭 1936 +夹 1937 +撞 1938 +刃 1939 +仰 1940 +阁 1941 +毅 1942 +郁 1943 +蕾 1944 +乎 1945 +愁 1946 +轿 1947 +吕 1948 +昨 1949 +椎 1950 +脉 1951 +巡 1952 +氨 1953 +烫 1954 +俱 1955 +岗 1956 +返 1957 +振 1958 +祸 1959 +串 1960 +蚕 1961 +勃 1962 +丛 1963 +莞 1964 +赢 1965 +谊 1966 +粥 1967 +帆 1968 +访 1969 +厕 1970 +诀 1971 +骂 1972 +奈 1973 +鉴 1974 +肺 1975 +辫 1976 +骏 1977 +葵 1978 +迟 1979 +赚 1980 +慈 1981 +函 1982 +铅 1983 +献 1984 +挡 1985 +驼 1986 +择 1987 +触 1988 +袁 1989 +晕 1990 +籽 1991 +冯 1992 +棍 1993 +协 1994 +邦 1995 +爬 1996 +糊 1997 +仲 1998 +桐 1999 +菩 2000 +郸 2001 +薯 2002 +够 2003 +径 2004 +税 2005 +斩 2006 +塑 2007 +胀 2008 +赌 2009 +锐 2010 +氓 2011 +隔 2012 +灿 2013 +浓 2014 +牵 2015 +碱 2016 +绒 2017 +肃 2018 +券 2019 +阔 2020 +檬 2021 +梭 2022 +丈 2023 +劲 2024 +啡 2025 +悬 2026 +坤 2027 +馅 2028 +瑜 2029 +呆 2030 +勿 2031 +虐 2032 +栏 2033 +哲 2034 +殿 2035 +缓 2036 +扩 2037 +恢 2038 +凶 2039 +纷 2040 +箫 2041 +讨 2042 +矮 2043 +碗 2044 +披 2045 +袖 2046 +嫂 2047 +械 2048 +侯 2049 +丧 2050 +责 2051 +噜 2052 +闺 2053 +茎 2054 +屠 2055 +夸 2056 +拥 2057 +扰 2058 +骤 2059 +筷 2060 +占 2061 +旦 2062 +屈 2063 +寞 2064 +贷 2065 +畅 2066 +媛 2067 +匹 2068 +暮 2069 +吨 2070 +邯 2071 +衫 2072 +渔 2073 +粮 2074 +贸 2075 +悄 2076 +灾 2077 +煎 2078 +唇 2079 +骗 2080 +逢 2081 +恨 2082 +淑 2083 +患 2084 +兑 2085 +惯 2086 +迈 2087 +卢 2088 +慕 2089 +恰 2090 +贫 2091 +阻 2092 +供 2093 +颂 2094 +咚 2095 +纲 2096 +蜀 2097 +蜗 2098 +掘 2099 +茂 2100 +储 2101 +藤 2102 +扁 2103 +麟 2104 +柠 2105 +骆 2106 +崇 2107 +泸 2108 +丢 2109 +螃 2110 +疮 2111 +享 2112 +秒 2113 +铝 2114 +仗 2115 +彤 2116 +恭 2117 +吾 2118 +贞 2119 +衰 2120 +斜 2121 +喉 2122 +晒 2123 +雁 2124 +穆 2125 +搬 2126 +磊 2127 +拆 2128 +悔 2129 +卿 2130 +浅 2131 +涨 2132 +硝 2133 +滋 2134 +崛 2135 +珊 2136 +茉 2137 +弓 2138 +湘 2139 +枫 2140 +踏 2141 +榴 2142 +寨 2143 +翰 2144 +吟 2145 +均 2146 +蝉 2147 +卸 2148 +嘎 2149 +愚 2150 +剖 2151 +哎 2152 +污 2153 +镖 2154 +遮 2155 +垫 2156 +勾 2157 +鸦 2158 +鲤 2159 +埃 2160 +替 2161 +忌 2162 +犹 2163 +劫 2164 +笛 2165 +咤 2166 +傅 2167 +绕 2168 +帐 2169 +鹦 2170 +狠 2171 +缝 2172 +遍 2173 +拾 2174 +巢 2175 +僧 2176 +煤 2177 +铭 2178 +厌 2179 +鹉 2180 +帕 2181 +伞 2182 +蹄 2183 +阶 2184 +佐 2185 +既 2186 +抚 2187 +吞 2188 +橘 2189 +跨 2190 +肩 2191 +摔 2192 +猩 2193 +凝 2194 +甫 2195 +旭 2196 +赖 2197 +瘤 2198 +羞 2199 +叹 2200 +弦 2201 +璐 2202 +臣 2203 +沃 2204 +忽 2205 +闷 2206 +钩 2207 +嬛 2208 +谋 2209 +豌 2210 +骄 2211 +昂 2212 +耐 2213 +鑫 2214 +审 2215 +另 2216 +呐 2217 +帘 2218 +舰 2219 +咒 2220 +栀 2221 +崔 2222 +顷 2223 +跃 2224 +罚 2225 +谎 2226 +奉 2227 +钉 2228 +瀑 2229 +障 2230 +诵 2231 +挥 2232 +驱 2233 +蟒 2234 +渴 2235 +蛮 2236 +枕 2237 +猿 2238 +砖 2239 +瑶 2240 +援 2241 +咙 2242 +稻 2243 +潇 2244 +孽 2245 +拨 2246 +刮 2247 +奴 2248 +辅 2249 +催 2250 +飙 2251 +蓬 2252 +腌 2253 +届 2254 +谣 2255 +踢 2256 +笋 2257 +陪 2258 +壶 2259 +且 2260 +姬 2261 +渐 2262 +予 2263 +幅 2264 +倩 2265 +椅 2266 +芙 2267 +逍 2268 +坠 2269 +驻 2270 +侵 2271 +濮 2272 +毁 2273 +鲸 2274 +靓 2275 +钾 2276 +洒 2277 +栽 2278 +伽 2279 +侏 2280 +爪 2281 +嘻 2282 +杞 2283 +允 2284 +楠 2285 +践 2286 +扶 2287 +轨 2288 +巫 2289 +谦 2290 +昏 2291 +邢 2292 +哑 2293 +爵 2294 +抹 2295 +苑 2296 +硕 2297 +漏 2298 +寡 2299 +啤 2300 +伙 2301 +扒 2302 +殊 2303 +尤 2304 +沫 2305 +贡 2306 +鞍 2307 +蚊 2308 +贪 2309 +蝠 2310 +藕 2311 +冈 2312 +兆 2313 +萱 2314 +梳 2315 +灌 2316 +仆 2317 +皆 2318 +咽 2319 +粱 2320 +蝙 2321 +襄 2322 +弗 2323 +缸 2324 +禹 2325 +咕 2326 +菏 2327 +您 2328 +宰 2329 +斋 2330 +慌 2331 +絮 2332 +痕 2333 +崖 2334 +韭 2335 +担 2336 +锈 2337 +黛 2338 +盈 2339 +稳 2340 +翩 2341 +呱 2342 +疏 2343 +嫩 2344 +轴 2345 +耽 2346 +熬 2347 +怨 2348 +盲 2349 +脾 2350 +详 2351 +却 2352 +顽 2353 +鲍 2354 +疫 2355 +溃 2356 +茫 2357 +闰 2358 +贱 2359 +鱿 2360 +扭 2361 +舅 2362 +痣 2363 +劝 2364 +浑 2365 +廷 2366 +迫 2367 +驯 2368 +挣 2369 +耗 2370 +碑 2371 +饶 2372 +雯 2373 +卑 2374 +锻 2375 +辈 2376 +哇 2377 +廊 2378 +契 2379 +赠 2380 +艇 2381 +辩 2382 +努 2383 +枸 2384 +枯 2385 +芜 2386 +绸 2387 +碟 2388 +潍 2389 +逐 2390 +咯 2391 +框 2392 +跆 2393 +竟 2394 +挺 2395 +娟 2396 +挠 2397 +尝 2398 +咱 2399 +蜓 2400 +赋 2401 +麒 2402 +姻 2403 +臂 2404 +拐 2405 +敲 2406 +牢 2407 +损 2408 +誉 2409 +祭 2410 +捡 2411 +荟 2412 +砍 2413 +乾 2414 +墅 2415 +砂 2416 +蜻 2417 +烛 2418 +婉 2419 +脖 2420 +饱 2421 +奕 2422 +吓 2423 +煲 2424 +授 2425 +耍 2426 +叙 2427 +斌 2428 +欺 2429 +呦 2430 +拽 2431 +橱 2432 +肛 2433 +舔 2434 +盾 2435 +梗 2436 +獒 2437 +遵 2438 +媒 2439 +秧 2440 +疲 2441 +勋 2442 +夷 2443 +陷 2444 +恼 2445 +睿 2446 +苔 2447 +彬 2448 +贯 2449 +榆 2450 +弥 2451 +茜 2452 +璧 2453 +馒 2454 +乒 2455 +诊 2456 +愈 2457 +棵 2458 +盼 2459 +兮 2460 +窖 2461 +霹 2462 +渊 2463 +嫡 2464 +歪 2465 +堆 2466 +胥 2467 +喵 2468 +雳 2469 +痔 2470 +醇 2471 +芹 2472 +杠 2473 +辱 2474 +裳 2475 +乓 2476 +虑 2477 +谐 2478 +薪 2479 +瑟 2480 +剩 2481 +戈 2482 +岚 2483 +锥 2484 +澜 2485 +逝 2486 +涩 2487 +敦 2488 +俪 2489 +姥 2490 +缠 2491 +揭 2492 +翘 2493 +轼 2494 +沁 2495 +吵 2496 +臀 2497 +琼 2498 +泼 2499 +挤 2500 +仑 2501 +惨 2502 +喽 2503 +姗 2504 +曝 2505 +韦 2506 +堰 2507 +辕 2508 +敷 2509 +兹 2510 +伪 2511 +纤 2512 +蚌 2513 +琦 2514 +淄 2515 +役 2516 +脊 2517 +匪 2518 +嗓 2519 +哀 2520 +豚 2521 +冤 2522 +衬 2523 +掩 2524 +措 2525 +卤 2526 +竿 2527 +钰 2528 +宴 2529 +稣 2530 +疡 2531 +擎 2532 +悉 2533 +喊 2534 +嘿 2535 +痴 2536 +捏 2537 +糯 2538 +廉 2539 +淀 2540 +狄 2541 +纽 2542 +娥 2543 +魁 2544 +犀 2545 +霄 2546 +彦 2547 +卦 2548 +祛 2549 +屯 2550 +裕 2551 +涕 2552 +邱 2553 +哮 2554 +础 2555 +邵 2556 +棱 2557 +厢 2558 +岷 2559 +屌 2560 +伐 2561 +浇 2562 +冥 2563 +巩 2564 +嗨 2565 +泄 2566 +炭 2567 +攀 2568 +沸 2569 +呵 2570 +痰 2571 +堵 2572 +榨 2573 +虽 2574 +悍 2575 +溺 2576 +甸 2577 +娶 2578 +拒 2579 +艰 2580 +泻 2581 +靖 2582 +萤 2583 +烽 2584 +燥 2585 +妓 2586 +赣 2587 +郊 2588 +锤 2589 +棠 2590 +煌 2591 +抛 2592 +脆 2593 +匠 2594 +鞠 2595 +焖 2596 +筹 2597 +遭 2598 +魄 2599 +鸳 2600 +喇 2601 +轰 2602 +抖 2603 +瞳 2604 +遂 2605 +欠 2606 +泌 2607 +瓣 2608 +梓 2609 +娅 2610 +拇 2611 +拯 2612 +埋 2613 +棚 2614 +鸯 2615 +循 2616 +怜 2617 +袍 2618 +昭 2619 +兼 2620 +挨 2621 +瘩 2622 +昕 2623 +硅 2624 +咨 2625 +抑 2626 +栗 2627 +呛 2628 +榄 2629 +滕 2630 +蔷 2631 +漯 2632 +疙 2633 +粑 2634 +搏 2635 +沿 2636 +握 2637 +巷 2638 +葬 2639 +螂 2640 +禽 2641 +膝 2642 +邑 2643 +绞 2644 +莺 2645 +峨 2646 +峙 2647 +俺 2648 +鄙 2649 +焊 2650 +倚 2651 +叭 2652 +酥 2653 +株 2654 +捞 2655 +倪 2656 +愉 2657 +瘾 2658 +绥 2659 +翡 2660 +脐 2661 +促 2662 +巅 2663 +抵 2664 +粤 2665 +俩 2666 +瞬 2667 +禾 2668 +蓄 2669 +曰 2670 +溢 2671 +绪 2672 +丙 2673 +沾 2674 +皂 2675 +酿 2676 +陋 2677 +绰 2678 +喘 2679 +椰 2680 +耻 2681 +裔 2682 +锌 2683 +儒 2684 +柚 2685 +惩 2686 +埠 2687 +囧 2688 +畏 2689 +佑 2690 +邹 2691 +灸 2692 +薰 2693 +坪 2694 +崩 2695 +肪 2696 +崎 2697 +漳 2698 +昵 2699 +酮 2700 +橄 2701 +鹃 2702 +呕 2703 +粽 2704 +猕 2705 +茹 2706 +峻 2707 +唉 2708 +呈 2709 +唢 2710 +佣 2711 +汕 2712 +削 2713 +恍 2714 +霖 2715 +梧 2716 +携 2717 +惧 2718 +誓 2719 +嗝 2720 +庞 2721 +曦 2722 +泗 2723 +沌 2724 +湛 2725 +瞎 2726 +暇 2727 +镯 2728 +啸 2729 +栩 2730 +窟 2731 +涉 2732 +鲫 2733 +伸 2734 +倦 2735 +谓 2736 +沐 2737 +刹 2738 +貂 2739 +耕 2740 +诡 2741 +氮 2742 +聋 2743 +淳 2744 +弘 2745 +霾 2746 +躲 2747 +佟 2748 +藻 2749 +纺 2750 +淹 2751 +靴 2752 +狭 2753 +槽 2754 +涌 2755 +庸 2756 +娲 2757 +谍 2758 +桔 2759 +磷 2760 +捣 2761 +焉 2762 +砸 2763 +赔 2764 +鳞 2765 +甩 2766 +渣 2767 +芈 2768 +债 2769 +蔚 2770 +庶 2771 +糟 2772 +篱 2773 +凸 2774 +睫 2775 +拘 2776 +挫 2777 +屑 2778 +哩 2779 +诛 2780 +跤 2781 +啼 2782 +蠢 2783 +栋 2784 +窦 2785 +哟 2786 +灶 2787 +皓 2788 +凭 2789 +晏 2790 +逛 2791 +辐 2792 +铲 2793 +辟 2794 +窍 2795 +俐 2796 +钏 2797 +坝 2798 +蝌 2799 +饲 2800 +堪 2801 +蚪 2802 +凿 2803 +乞 2804 +矩 2805 +鳅 2806 +泣 2807 +镁 2808 +颠 2809 +劈 2810 +珀 2811 +矛 2812 +娄 2813 +氟 2814 +扔 2815 +皱 2816 +窃 2817 +裹 2818 +渠 2819 +哄 2820 +鹊 2821 +褒 2822 +绯 2823 +汝 2824 +叽 2825 +剥 2826 +孜 2827 +楂 2828 +粪 2829 +愧 2830 +斧 2831 +锯 2832 +仕 2833 +昔 2834 +阑 2835 +晰 2836 +缤 2837 +孢 2838 +羚 2839 +抬 2840 +屿 2841 +澄 2842 +撑 2843 +酵 2844 +糜 2845 +鄂 2846 +飓 2847 +芯 2848 +薏 2849 +渭 2850 +昊 2851 +虞 2852 +疤 2853 +暂 2854 +韶 2855 +惟 2856 +熏 2857 +坟 2858 +畔 2859 +澈 2860 +伍 2861 +坞 2862 +篷 2863 +慎 2864 +躺 2865 +衩 2866 +茨 2867 +嫦 2868 +凳 2869 +钧 2870 +馗 2871 +竭 2872 +孵 2873 +俯 2874 +籁 2875 +腥 2876 +躬 2877 +汾 2878 +琵 2879 +咆 2880 +偿 2881 +胳 2882 +荔 2883 +芋 2884 +蔽 2885 +贬 2886 +琶 2887 +羡 2888 +旷 2889 +炅 2890 +苯 2891 +酶 2892 +闫 2893 +稍 2894 +阎 2895 +尹 2896 +螳 2897 +邀 2898 +骐 2899 +膊 2900 +颁 2901 +鹂 2902 +帖 2903 +涡 2904 +兜 2905 +缅 2906 +嘞 2907 +躁 2908 +祁 2909 +茵 2910 +估 2911 +郝 2912 +妒 2913 +秤 2914 +焚 2915 +耿 2916 +勉 2917 +抠 2918 +晃 2919 +彪 2920 +滤 2921 +莽 2922 +晖 2923 +瞧 2924 +履 2925 +禅 2926 +矣 2927 +伶 2928 +碘 2929 +懈 2930 +彻 2931 +甚 2932 +葩 2933 +珑 2934 +袄 2935 +娆 2936 +驹 2937 +敞 2938 +唑 2939 +馄 2940 +滥 2941 +腚 2942 +滔 2943 +肇 2944 +仅 2945 +栓 2946 +纬 2947 +囚 2948 +棕 2949 +琅 2950 +韬 2951 +雌 2952 +詹 2953 +凹 2954 +掏 2955 +樊 2956 +抒 2957 +叛 2958 +卉 2959 +跌 2960 +朔 2961 +萎 2962 +俑 2963 +嚣 2964 +赴 2965 +冉 2966 +剃 2967 +哗 2968 +捐 2969 +搅 2970 +饨 2971 +宪 2972 +缕 2973 +颤 2974 +刁 2975 +趾 2976 +渤 2977 +寇 2978 +蛤 2979 +蝇 2980 +毯 2981 +荨 2982 +檀 2983 +摊 2984 +锰 2985 +绅 2986 +殷 2987 +颐 2988 +沥 2989 +捆 2990 +旱 2991 +釜 2992 +龈 2993 +拓 2994 +洱 2995 +勺 2996 +癫 2997 +泛 2998 +衢 2999 +寝 3000 +跪 3001 +隶 3002 +槐 3003 +鸥 3004 +妄 3005 +歉 3006 +亨 3007 +沽 3008 +揉 3009 +簧 3010 +泵 3011 +嚼 3012 +猬 3013 +癣 3014 +肘 3015 +婶 3016 +蜈 3017 +亏 3018 +滁 3019 +恤 3020 +嗜 3021 +瀚 3022 +呗 3023 +汹 3024 +羹 3025 +稚 3026 +蕊 3027 +蕴 3028 +俭 3029 +禧 3030 +妾 3031 +亩 3032 +枭 3033 +煞 3034 +芸 3035 +镐 3036 +憋 3037 +苟 3038 +窥 3039 +掠 3040 +纠 3041 +栈 3042 +酚 3043 +鹭 3044 +莆 3045 +栖 3046 +胰 3047 +辙 3048 +嫉 3049 +蚣 3050 +尧 3051 +朦 3052 +乏 3053 +沪 3054 +慨 3055 +庚 3056 +诈 3057 +昧 3058 +疱 3059 +弈 3060 +烷 3061 +龚 3062 +漾 3063 +羲 3064 +谨 3065 +撇 3066 +澎 3067 +侍 3068 +佰 3069 +浊 3070 +窄 3071 +讽 3072 +蔗 3073 +沛 3074 +羯 3075 +凛 3076 +噬 3077 +吁 3078 +雍 3079 +腻 3080 +矫 3081 +涧 3082 +箍 3083 +祺 3084 +崽 3085 +秃 3086 +罕 3087 +蔻 3088 +癜 3089 +塌 3090 +纶 3091 +烯 3092 +胧 3093 +恳 3094 +奎 3095 +嵩 3096 +碌 3097 +蔼 3098 +鞘 3099 +烙 3100 +亥 3101 +啄 3102 +瞻 3103 +仍 3104 +酯 3105 +浣 3106 +荧 3107 +焕 3108 +畜 3109 +椿 3110 +拙 3111 +璨 3112 +盔 3113 +隧 3114 +押 3115 +肢 3116 +廖 3117 +拂 3118 +瑕 3119 +渺 3120 +璀 3121 +婿 3122 +匀 3123 +蔓 3124 +腕 3125 +骷 3126 +梆 3127 +哔 3128 +腋 3129 +樟 3130 +锣 3131 +矢 3132 +碍 3133 +瘙 3134 +茸 3135 +潢 3136 +潼 3137 +祈 3138 +溴 3139 +嫌 3140 +脓 3141 +杖 3142 +漓 3143 +埔 3144 +瑚 3145 +衷 3146 +闽 3147 +蚓 3148 +歧 3149 +丞 3150 +郡 3151 +汀 3152 +睁 3153 +肆 3154 +撼 3155 +荫 3156 +蹲 3157 +灼 3158 +朽 3159 +豁 3160 +撤 3161 +馍 3162 +蚯 3163 +琉 3164 +亢 3165 +捧 3166 +烁 3167 +阱 3168 +隙 3169 +溅 3170 +棺 3171 +熔 3172 +劣 3173 +磅 3174 +铛 3175 +趁 3176 +呜 3177 +雏 3178 +旨 3179 +俞 3180 +钡 3181 +稽 3182 +冀 3183 +宛 3184 +璇 3185 +巍 3186 +枚 3187 +沭 3188 +刊 3189 +奢 3190 +擅 3191 +诲 3192 +搓 3193 +渍 3194 +咔 3195 +垣 3196 +奄 3197 +丐 3198 +趋 3199 +蟾 3200 +衔 3201 +禄 3202 +瓢 3203 +瞿 3204 +苞 3205 +桨 3206 +髅 3207 +趴 3208 +凄 3209 +绽 3210 +噩 3211 +狡 3212 +嚏 3213 +鳝 3214 +琐 3215 +拢 3216 +敖 3217 +韧 3218 +骇 3219 +壹 3220 +镑 3221 +稼 3222 +搁 3223 +棘 3224 +闸 3225 +蛰 3226 +哨 3227 +猥 3228 +纨 3229 +笆 3230 +淤 3231 +亳 3232 +鲈 3233 +倘 3234 +疣 3235 +擒 3236 +跷 3237 +蟆 3238 +楷 3239 +妲 3240 +嘲 3241 +瑙 3242 +冶 3243 +壤 3244 +蜒 3245 +莴 3246 +婵 3247 +玖 3248 +苓 3249 +戎 3250 +喧 3251 +蜥 3252 +弧 3253 +垢 3254 +酪 3255 +桦 3256 +喱 3257 +哼 3258 +髓 3259 +阀 3260 +忻 3261 +牲 3262 +拱 3263 +噪 3264 +赐 3265 +谅 3266 +兀 3267 +胚 3268 +粟 3269 +蜿 3270 +缉 3271 +绮 3272 +蛾 3273 +哉 3274 +嬉 3275 +窑 3276 +疸 3277 +钦 3278 +倔 3279 +岂 3280 +鼹 3281 +喋 3282 +玮 3283 +贩 3284 +蛹 3285 +芪 3286 +佬 3287 +酬 3288 +缪 3289 +汶 3290 +裆 3291 +枉 3292 +聂 3293 +腮 3294 +侨 3295 +甥 3296 +昼 3297 +钝 3298 +娩 3299 +暨 3300 +炊 3301 +凋 3302 +峪 3303 +嫖 3304 +羔 3305 +筱 3306 +磕 3307 +瞪 3308 +芥 3309 +陛 3310 +刨 3311 +蚀 3312 +蒿 3313 +耸 3314 +胺 3315 +艘 3316 +粼 3317 +萋 3318 +吼 3319 +烹 3320 +溧 3321 +钗 3322 +喔 3323 +筛 3324 +倡 3325 +谏 3326 +黔 3327 +锄 3328 +蟀 3329 +炳 3330 +琥 3331 +驿 3332 +睾 3333 +衍 3334 +椭 3335 +蟋 3336 +瘫 3337 +荚 3338 +渝 3339 +峦 3340 +眨 3341 +襟 3342 +熹 3343 +锵 3344 +傍 3345 +绔 3346 +缴 3347 +堤 3348 +醛 3349 +涟 3350 +傣 3351 +荞 3352 +觅 3353 +慷 3354 +挽 3355 +斛 3356 +靡 3357 +烘 3358 +浸 3359 +瘁 3360 +泾 3361 +蟑 3362 +榕 3363 +藿 3364 +祷 3365 +琢 3366 +褐 3367 +嗡 3368 +曙 3369 +蜇 3370 +妊 3371 +驭 3372 +辖 3373 +吱 3374 +遐 3375 +痧 3376 +笠 3377 +炙 3378 +擀 3379 +讶 3380 +咧 3381 +鲅 3382 +扯 3383 +缇 3384 +趟 3385 +嘱 3386 +秉 3387 +颇 3388 +苇 3389 +憾 3390 +恙 3391 +拦 3392 +钞 3393 +磐 3394 +郴 3395 +峭 3396 +攘 3397 +挪 3398 +痱 3399 +剿 3400 +陇 3401 +稠 3402 +坂 3403 +叨 3404 +牺 3405 +晟 3406 +吝 3407 +殇 3408 +匿 3409 +陨 3410 +茧 3411 +绚 3412 +蝈 3413 +裴 3414 +闵 3415 +缆 3416 +梢 3417 +盏 3418 +诶 3419 +鸠 3420 +摧 3421 +铸 3422 +掀 3423 +嘀 3424 +鳖 3425 +垒 3426 +锲 3427 +扳 3428 +揽 3429 +沦 3430 +坎 3431 +桩 3432 +竽 3433 +囵 3434 +彰 3435 +狩 3436 +阡 3437 +袅 3438 +尬 3439 +蛊 3440 +呻 3441 +铂 3442 +膨 3443 +舜 3444 +庵 3445 +槟 3446 +霓 3447 +辘 3448 +镶 3449 +磺 3450 +凑 3451 +痞 3452 +芷 3453 +匕 3454 +唧 3455 +喳 3456 +嗦 3457 +斥 3458 +漩 3459 +逵 3460 +炯 3461 +牟 3462 +岐 3463 +穗 3464 +梵 3465 +逮 3466 +萃 3467 +芮 3468 +尴 3469 +煸 3470 +呸 3471 +阙 3472 +氛 3473 +蹑 3474 +鹌 3475 +鸵 3476 +胭 3477 +鹑 3478 +枇 3479 +璋 3480 +蜴 3481 +囫 3482 +舂 3483 +祠 3484 +糙 3485 +恬 3486 +悯 3487 +汰 3488 +尉 3489 +纫 3490 +彝 3491 +挚 3492 +淌 3493 +盎 3494 +羿 3495 +蓟 3496 +竺 3497 +瞩 3498 +杷 3499 +昙 3500 +朕 3501 +惶 3502 +惰 3503 +婕 3504 +傀 3505 +炽 3506 +儡 3507 +簇 3508 +绎 3509 +秽 3510 +弛 3511 +迂 3512 +怦 3513 +罢 3514 +腩 3515 +喰 3516 +蜕 3517 +荠 3518 +啬 3519 +逻 3520 +弩 3521 +榻 3522 +婺 3523 +迢 3524 +暧 3525 +犊 3526 +筐 3527 +烨 3528 +鲟 3529 +惭 3530 +涿 3531 +毙 3532 +涣 3533 +鲶 3534 +痫 3535 +诅 3536 +沼 3537 +黏 3538 +禺 3539 +虱 3540 +卒 3541 +喀 3542 +淅 3543 +皑 3544 +堕 3545 +黯 3546 +咐 3547 +墩 3548 +剁 3549 +莘 3550 +擂 3551 +簪 3552 +睹 3553 +斓 3554 +缭 3555 +犁 3556 +肋 3557 +挎 3558 +挞 3559 +崭 3560 +夭 3561 +厄 3562 +蹭 3563 +妥 3564 +圃 3565 +唠 3566 +遣 3567 +茯 3568 +泷 3569 +徊 3570 +砰 3571 +辜 3572 +饪 3573 +秩 3574 +帷 3575 +帜 3576 +铵 3577 +菁 3578 +俘 3579 +旬 3580 +痿 3581 +叼 3582 +褪 3583 +眺 3584 +徘 3585 +屡 3586 +檐 3587 +猾 3588 +戳 3589 +赎 3590 +肴 3591 +逅 3592 +奂 3593 +橇 3594 +皋 3595 +岔 3596 +邂 3597 +芊 3598 +瑾 3599 +茗 3600 +骥 3601 +睦 3602 +淼 3603 +滦 3604 +砌 3605 +邳 3606 +胁 3607 +驸 3608 +侄 3609 +署 3610 +尻 3611 +蝗 3612 +癞 3613 +剔 3614 +汐 3615 +冕 3616 +歼 3617 +仞 3618 +蚱 3619 +翟 3620 +湄 3621 +寐 3622 +睢 3623 +茬 3624 +炬 3625 +懿 3626 +掰 3627 +琊 3628 +陡 3629 +镰 3630 +鬓 3631 +腱 3632 +僻 3633 +绢 3634 +墟 3635 +孺 3636 +娓 3637 +毽 3638 +绊 3639 +馈 3640 +皖 3641 +跋 3642 +啃 3643 +珂 3644 +怯 3645 +饵 3646 +蛟 3647 +寥 3648 +孚 3649 +峥 3650 +阮 3651 +撩 3652 +馋 3653 +嘘 3654 +杵 3655 +恕 3656 +诫 3657 +揍 3658 +嶙 3659 +胯 3660 +粹 3661 +眩 3662 +蚝 3663 +煜 3664 +诧 3665 +鳕 3666 +垠 3667 +娼 3668 +铢 3669 +伺 3670 +瘟 3671 +蔺 3672 +胱 3673 +虏 3674 +烩 3675 +娴 3676 +飒 3677 +惚 3678 +涅 3679 +峋 3680 +雇 3681 +琛 3682 +窕 3683 +懊 3684 +晾 3685 +洙 3686 +屹 3687 +馁 3688 +苣 3689 +掐 3690 +眯 3691 +柄 3692 +濒 3693 +嚷 3694 +鳌 3695 +窈 3696 +钛 3697 +栾 3698 +窜 3699 +婊 3700 +榔 3701 +疝 3702 +裘 3703 +蓓 3704 +幄 3705 +霏 3706 +捶 3707 +郫 3708 +蘸 3709 +眷 3710 +颍 3711 +脯 3712 +挛 3713 +鹳 3714 +钮 3715 +嗒 3716 +糠 3717 +桓 3718 +骼 3719 +噼 3720 +葚 3721 +抉 3722 +侮 3723 +匡 3724 +洼 3725 +咀 3726 +惫 3727 +眸 3728 +婪 3729 +忐 3730 +蛐 3731 +芍 3732 +拧 3733 +莒 3734 +蚤 3735 +悚 3736 +坻 3737 +矾 3738 +忑 3739 +悼 3740 +壑 3741 +肽 3742 +蓑 3743 +攒 3744 +玟 3745 +鲢 3746 +歹 3747 +吭 3748 +痉 3749 +蛎 3750 +骁 3751 +娠 3752 +罂 3753 +缚 3754 +捅 3755 +瘸 3756 +舱 3757 +遁 3758 +寅 3759 +雹 3760 +珞 3761 +簸 3762 +酌 3763 +枢 3764 +蕨 3765 +簿 3766 +偃 3767 +隽 3768 +褂 3769 +轲 3770 +缀 3771 +钳 3772 +颊 3773 +戮 3774 +嵊 3775 +掷 3776 +涤 3777 +侃 3778 +嫔 3779 +瞄 3780 +膛 3781 +赁 3782 +癖 3783 +讳 3784 +皎 3785 +忡 3786 +隍 3787 +膳 3788 +搔 3789 +讼 3790 +唰 3791 +滟 3792 +冽 3793 +孰 3794 +兢 3795 +漱 3796 +躯 3797 +雎 3798 +嘹 3799 +捂 3800 +婧 3801 +蔑 3802 +眈 3803 +糗 3804 +镀 3805 +兖 3806 +鬟 3807 +唾 3808 +刿 3809 +羌 3810 +畸 3811 +涮 3812 +迭 3813 +榛 3814 +熄 3815 +礴 3816 +殴 3817 +熠 3818 +擞 3819 +矶 3820 +茴 3821 +颅 3822 +瓯 3823 +鳗 3824 +匣 3825 +滞 3826 +廓 3827 +痦 3828 +邋 3829 +戊 3830 +侈 3831 +惋 3832 +骅 3833 +岖 3834 +嶂 3835 +搂 3836 +敛 3837 +燎 3838 +砚 3839 +吏 3840 +椟 3841 +渲 3842 +螨 3843 +蛀 3844 +遢 3845 +噢 3846 +毓 3847 +蹊 3848 +锂 3849 +沱 3850 +崴 3851 +鹬 3852 +乍 3853 +奠 3854 +庾 3855 +焯 3856 +啰 3857 +鸢 3858 +扉 3859 +霎 3860 +耙 3861 +茁 3862 +懦 3863 +阆 3864 +柬 3865 +妨 3866 +矜 3867 +靶 3868 +蜃 3869 +湃 3870 +驳 3871 +唳 3872 +佗 3873 +崂 3874 +浜 3875 +舵 3876 +募 3877 +骛 3878 +瘀 3879 +馏 3880 +扛 3881 +槿 3882 +屄 3883 +蹬 3884 +茱 3885 +眶 3886 +屉 3887 +嚎 3888 +疚 3889 +涸 3890 +歆 3891 +啵 3892 +猝 3893 +藓 3894 +骊 3895 +胤 3896 +渗 3897 +铮 3898 +憧 3899 +鞅 3900 +颓 3901 +礁 3902 +悴 3903 +憬 3904 +嘚 3905 +曳 3906 +踌 3907 +酰 3908 +帚 3909 +墉 3910 +嵌 3911 +楞 3912 +砀 3913 +铿 3914 +喃 3915 +揣 3916 +筠 3917 +潋 3918 +曜 3919 +吩 3920 +徙 3921 +岱 3922 +缎 3923 +拷 3924 +憎 3925 +谛 3926 +铉 3927 +躇 3928 +坨 3929 +憔 3930 +觑 3931 +铡 3932 +紊 3933 +腑 3934 +唏 3935 +硒 3936 +撅 3937 +濡 3938 +苷 3939 +殡 3940 +妩 3941 +鹫 3942 +嗪 3943 +柑 3944 +斐 3945 +聆 3946 +郓 3947 +衙 3948 +浚 3949 +匈 3950 +涞 3951 +潺 3952 +戬 3953 +罔 3954 +坷 3955 +樵 3956 +荤 3957 +涓 3958 +酝 3959 +蹋 3960 +蠡 3961 +逾 3962 +掺 3963 +纣 3964 +弑 3965 +嗷 3966 +貔 3967 +钊 3968 +嬷 3969 +炝 3970 +镂 3971 +谴 3972 +忏 3973 +篆 3974 +砺 3975 +嘶 3976 +瞒 3977 +潦 3978 +狞 3979 +胗 3980 +妣 3981 +脍 3982 +呲 3983 +狈 3984 +垄 3985 +咩 3986 +怠 3987 +翱 3988 +瓮 3989 +掖 3990 +骞 3991 +愣 3992 +貅 3993 +盯 3994 +惬 3995 +驮 3996 +讥 3997 +噗 3998 +戟 3999 +亵 4000 +饯 4001 +葭 4002 +颦 4003 +卯 4004 +踵 4005 +臻 4006 +泓 4007 +肮 4008 +茼 4009 +浔 4010 +荥 4011 +鄞 4012 +洽 4013 +嗅 4014 +碾 4015 +疵 4016 +澹 4017 +绷 4018 +涪 4019 +桢 4020 +垦 4021 +璞 4022 +叱 4023 +耒 4024 +瑛 4025 +炕 4026 +沮 4027 +潸 4028 +蒹 4029 +婢 4030 +麝 4031 +攸 4032 +炜 4033 +蕙 4034 +鄢 4035 +岑 4036 +恃 4037 +鄱 4038 +拣 4039 +戌 4040 +珲 4041 +窿 4042 +呤 4043 +揪 4044 +羁 4045 +晦 4046 +逞 4047 +摁 4048 +蒡 4049 +殃 4050 +宦 4051 +瞌 4052 +轶 4053 +隅 4054 +掬 4055 +祀 4056 +啉 4057 +噎 4058 +迩 4059 +豇 4060 +汞 4061 +咎 4062 +陟 4063 +毡 4064 +痹 4065 +遨 4066 +铐 4067 +赘 4068 +矗 4069 +砥 4070 +跄 4071 +熨 4072 +漪 4073 +鸾 4074 +绫 4075 +舆 4076 +吠 4077 +犸 4078 +柘 4079 +宸 4080 +陂 4081 +泫 4082 +邸 4083 +萸 4084 +藩 4085 +枰 4086 +撵 4087 +萦 4088 +拴 4089 +绛 4090 +澧 4091 +匮 4092 +轧 4093 +诣 4094 +怅 4095 +盂 4096 +馥 4097 +玥 4098 +垮 4099 +秸 4100 +踉 4101 +荀 4102 +嵘 4103 +暝 4104 +汲 4105 +郯 4106 +耘 4107 +踊 4108 +溯 4109 +瘊 4110 +腼 4111 +町 4112 +倭 4113 +捺 4114 +睬 4115 +踞 4116 +惕 4117 +缔 4118 +篓 4119 +彗 4120 +烀 4121 +辗 4122 +腆 4123 +鲳 4124 +麋 4125 +诃 4126 +沓 4127 +跚 4128 +杳 4129 +姹 4130 +萁 4131 +咫 4132 +稞 4133 +猖 4134 +茏 4135 +竣 4136 +稷 4137 +遛 4138 +厥 4139 +揠 4140 +膺 4141 +煦 4142 +帼 4143 +咻 4144 +挟 4145 +锭 4146 +唔 4147 +敕 4148 +踝 4149 +苋 4150 +麓 4151 +秆 4152 +臊 4153 +蹒 4154 +渎 4155 +赦 4156 +葆 4157 +泞 4158 +驷 4159 +蟠 4160 +诏 4161 +溉 4162 +臧 4163 +濑 4164 +烃 4165 +懵 4166 +藁 4167 +狰 4168 +谧 4169 +湍 4170 +惺 4171 +姊 4172 +捎 4173 +鹜 4174 +敝 4175 +翎 4176 +涎 4177 +勘 4178 +筏 4179 +噶 4180 +痢 4181 +酞 4182 +酣 4183 +缰 4184 +圭 4185 +迥 4186 +潞 4187 +焗 4188 +撬 4189 +邃 4190 +浃 4191 +殆 4192 +嗑 4193 +侬 4194 +骡 4195 +鹄 4196 +酋 4197 +蛆 4198 +偕 4199 +痤 4200 +斟 4201 +豉 4202 +戛 4203 +窘 4204 +碣 4205 +悸 4206 +骋 4207 +趵 4208 +栅 4209 +逑 4210 +钴 4211 +笈 4212 +瞰 4213 +樽 4214 +吆 4215 +榭 4216 +壕 4217 +寮 4218 +拈 4219 +沅 4220 +伫 4221 +扦 4222 +渚 4223 +惦 4224 +舶 4225 +痪 4226 +啧 4227 +褶 4228 +谆 4229 +珈 4230 +庇 4231 +汛 4232 +芩 4233 +婀 4234 +褴 4235 +殉 4236 +佘 4237 +奚 4238 +啾 4239 +荜 4240 +惴 4241 +仟 4242 +褛 4243 +姝 4244 +遏 4245 +搽 4246 +拮 4247 +忾 4248 +奘 4249 +缨 4250 +盱 4251 +洮 4252 +镭 4253 +魇 4254 +蕲 4255 +郦 4256 +蚩 4257 +踹 4258 +泯 4259 +砣 4260 +谬 4261 +昀 4262 +珉 4263 +眙 4264 +贿 4265 +嘈 4266 +褚 4267 +搐 4268 +枷 4269 +硼 4270 +拎 4271 +釉 4272 +鼾 4273 +箔 4274 +帛 4275 +醴 4276 +讹 4277 +惮 4278 +瞅 4279 +孪 4280 +喆 4281 +呋 4282 +郏 4283 +拭 4284 +漉 4285 +糍 4286 +讷 4287 +飚 4288 +瑄 4289 +霁 4290 +咦 4291 +枥 4292 +臼 4293 +祯 4294 +鲛 4295 +蕃 4296 +幢 4297 +笃 4298 +醪 4299 +獭 4300 +邛 4301 +鼬 4302 +瘪 4303 +嗖 4304 +黝 4305 +谙 4306 +嘭 4307 +瘘 4308 +娣 4309 +轱 4310 +瘠 4311 +蠕 4312 +匾 4313 +聩 4314 +歙 4315 +龌 4316 +鳍 4317 +冢 4318 +镕 4319 +荼 4320 +轳 4321 +弋 4322 +胫 4323 +昱 4324 +毋 4325 +缙 4326 +朐 4327 +侥 4328 +栉 4329 +骸 4330 +碉 4331 +讴 4332 +虔 4333 +徇 4334 +濯 4335 +摹 4336 +裱 4337 +庖 4338 +鸪 4339 +怏 4340 +贻 4341 +寰 4342 +叟 4343 +豺 4344 +恪 4345 +暄 4346 +铎 4347 +泱 4348 +浠 4349 +撰 4350 +鬣 4351 +龊 4352 +氰 4353 +札 4354 +蛭 4355 +棣 4356 +痊 4357 +箕 4358 +痂 4359 +哌 4360 +咄 4361 +侗 4362 +蛔 4363 +衅 4364 +妤 4365 +颔 4366 +岫 4367 +炔 4368 +袤 4369 +颚 4370 +宥 4371 +锉 4372 +啶 4373 +偻 4374 +颌 4375 +仃 4376 +扈 4377 +苒 4378 +蹴 4379 +苛 4380 +牍 4381 +骰 4382 +惇 4383 +咿 4384 +伎 4385 +皿 4386 +榈 4387 +缈 4388 +辄 4389 +鹧 4390 +晁 4391 +叩 4392 +崃 4393 +荏 4394 +怆 4395 +蜍 4396 +赉 4397 +滇 4398 +惘 4399 +溥 4400 +儆 4401 +髻 4402 +牦 4403 +沏 4404 +瑗 4405 +蹉 4406 +淞 4407 +渑 4408 +箩 4409 +槛 4410 +蓦 4411 +挝 4412 +楣 4413 +跎 4414 +氩 4415 +铤 4416 +蜊 4417 +芡 4418 +汴 4419 +咛 4420 +绌 4421 +淖 4422 +炀 4423 +祟 4424 +睽 4425 +苜 4426 +灏 4427 +蔫 4428 +菡 4429 +蓿 4430 +孑 4431 +舀 4432 +偎 4433 +坯 4434 +唻 4435 +缥 4436 +怂 4437 +曛 4438 +褥 4439 +毂 4440 +瞑 4441 +戍 4442 +玷 4443 +撮 4444 +莜 4445 +讪 4446 +赡 4447 +鹩 4448 +徨 4449 +罡 4450 +荸 4451 +皙 4452 +跺 4453 +彷 4454 +噔 4455 +荃 4456 +灞 4457 +狒 4458 +啕 4459 +赳 4460 +焱 4461 +烬 4462 +拗 4463 +篡 4464 +袱 4465 +濂 4466 +弼 4467 +唬 4468 +赃 4469 +囱 4470 +滂 4471 +鄄 4472 +惆 4473 +谤 4474 +搀 4475 +阐 4476 +嚓 4477 +圩 4478 +蹂 4479 +骜 4480 +铄 4481 +膻 4482 +藐 4483 +畴 4484 +磋 4485 +獾 4486 +甯 4487 +邈 4488 +佝 4489 +禀 4490 +嗤 4491 +桀 4492 +钎 4493 +羟 4494 +醺 4495 +卞 4496 +酐 4497 +颧 4498 +吡 4499 +狍 4500 +酉 4501 +垛 4502 +涝 4503 +臾 4504 +胍 4505 +锚 4506 +蜚 4507 +貉 4508 +愕 4509 +锢 4510 +酊 4511 +蜉 4512 +迤 4513 +铣 4514 +揖 4515 +钨 4516 +覃 4517 +晌 4518 +掣 4519 +凇 4520 +匝 4521 +栎 4522 +阉 4523 +汨 4524 +蚬 4525 +俨 4526 +觞 4527 +鳜 4528 +恻 4529 +钣 4530 +谕 4531 +醚 4532 +仨 4533 +铬 4534 +刈 4535 +衿 4536 +贰 4537 +窒 4538 +丕 4539 +犟 4540 +躏 4541 +汆 4542 +饴 4543 +厮 4544 +簌 4545 +饕 4546 +嗣 4547 +扼 4548 +儋 4549 +睑 4550 +驽 4551 +堑 4552 +娑 4553 +隼 4554 +濠 4555 +忪 4556 +噻 4557 +漕 4558 +茕 4559 +缄 4560 +锹 4561 +俎 4562 +螈 4563 +亘 4564 +镍 4565 +瀛 4566 +绦 4567 +岌 4568 +戾 4569 +胛 4570 +翊 4571 +篑 4572 +芎 4573 +銮 4574 +嘌 4575 +毖 4576 +掂 4577 +贮 4578 +幡 4579 +煊 4580 +邡 4581 +劵 4582 +濉 4583 +髦 4584 +茌 4585 +囤 4586 +夙 4587 +蜢 4588 +诠 4589 +黍 4590 +恣 4591 +诬 4592 +氦 4593 +舐 4594 +睐 4595 +谗 4596 +嘣 4597 +肟 4598 +鲲 4599 +塾 4600 +肏 4601 +砾 4602 +罄 4603 +苁 4604 +艹 4605 +玑 4606 +旌 4607 +鲽 4608 +搪 4609 +捍 4610 +逶 4611 +阖 4612 +煽 4613 +殚 4614 +伥 4615 +耷 4616 +叁 4617 +枞 4618 +嗟 4619 +芨 4620 +辍 4621 +踮 4622 +蛳 4623 +箬 4624 +屐 4625 +娌 4626 +箴 4627 +蝾 4628 +柒 4629 +琨 4630 +犄 4631 +蚜 4632 +膑 4633 +涔 4634 +槌 4635 +裟 4636 +萘 4637 +菟 4638 +嬴 4639 +琍 4640 +忱 4641 +靳 4642 +髋 4643 +旮 4644 +抻 4645 +餮 4646 +郧 4647 +飕 4648 +涠 4649 +煅 4650 +饽 4651 +眦 4652 +噤 4653 +茭 4654 +迄 4655 +苄 4656 +箪 4657 +诽 4658 +掸 4659 +谀 4660 +悖 4661 +瓤 4662 +聿 4663 +憩 4664 +谩 4665 +疖 4666 +藜 4667 +阚 4668 +蛏 4669 +壬 4670 +肱 4671 +妯 4672 +篙 4673 +蜱 4674 +垩 4675 +樨 4676 +悌 4677 +囔 4678 +曈 4679 +脲 4680 +啫 4681 +彧 4682 +碴 4683 +赊 4684 +篝 4685 +鳃 4686 +霭 4687 +呷 4688 +焙 4689 +溏 4690 +萼 4691 +徉 4692 +倨 4693 +砒 4694 +坳 4695 +桧 4696 +颢 4697 +拄 4698 +颉 4699 +砭 4700 +镳 4701 +桅 4702 +伢 4703 +燮 4704 +缛 4705 +裨 4706 +粕 4707 +臃 4708 +魍 4709 +蝣 4710 +荇 4711 +疥 4712 +唛 4713 +愠 4714 +麸 4715 +蜷 4716 +徜 4717 +魉 4718 +耦 4719 +樾 4720 +瓴 4721 +竦 4722 +扪 4723 +嵬 4724 +袈 4725 +谡 4726 +秣 4727 +晷 4728 +浥 4729 +宕 4730 +哧 4731 +攥 4732 +瞥 4733 +坍 4734 +怔 4735 +犷 4736 +摞 4737 +撷 4738 +熘 4739 +巽 4740 +湟 4741 +倜 4742 +鳊 4743 +鬃 4744 +磬 4745 +傥 4746 +菈 4747 +捻 4748 +畦 4749 +翳 4750 +侑 4751 +厝 4752 +洵 4753 +瞠 4754 +臆 4755 +忒 4756 +跬 4757 +璜 4758 +醍 4759 +吮 4760 +蒌 4761 +邺 4762 +僚 4763 +虬 4764 +蹿 4765 +綦 4766 +羧 4767 +诘 4768 +姣 4769 +楹 4770 +忸 4771 +铆 4772 +邬 4773 +踱 4774 +隘 4775 +怩 4776 +滢 4777 +喑 4778 +魑 4779 +龇 4780 +冼 4781 +哽 4782 +譬 4783 +迸 4784 +仄 4785 +赅 4786 +邰 4787 +淙 4788 +旯 4789 +莠 4790 +蹩 4791 +晤 4792 +诙 4793 +谟 4794 +佞 4795 +雉 4796 +砗 4797 +磲 4798 +誊 4799 +钒 4800 +膘 4801 +旖 4802 +匍 4803 +慑 4804 +恹 4805 +暹 4806 +踽 4807 +馕 4808 +晔 4809 +淆 4810 +崆 4811 +刎 4812 +纭 4813 +罘 4814 +桉 4815 +杈 4816 +淝 4817 +菅 4818 +鳐 4819 +谪 4820 +肓 4821 +盹 4822 +殒 4823 +旎 4824 +掼 4825 +柞 4826 +鲮 4827 +癀 4828 +岈 4829 +黜 4830 +铳 4831 +甬 4832 +匐 4833 +嗔 4834 +琰 4835 +犒 4836 +绉 4837 +闾 4838 +醐 4839 +蓥 4840 +媲 4841 +陉 4842 +蹶 4843 +髫 4844 +篁 4845 +勐 4846 +钵 4847 +笤 4848 +猗 4849 +皈 4850 +埂 4851 +泮 4852 +哓 4853 +腭 4854 +窠 4855 +隰 4856 +悱 4857 +芘 4858 +铰 4859 +芾 4860 +舫 4861 +摒 4862 +胄 4863 +嗲 4864 +骶 4865 +湮 4866 +诨 4867 +巳 4868 +靛 4869 +崮 4870 +盅 4871 +唆 4872 +鸬 4873 +菖 4874 +袒 4875 +鲑 4876 +砝 4877 +佚 4878 +倏 4879 +珩 4880 +痍 4881 +孬 4882 +绾 4883 +宓 4884 +坼 4885 +聒 4886 +赂 4887 +峒 4888 +擢 4889 +趄 4890 +啜 4891 +橼 4892 +衮 4893 +帧 4894 +螯 4895 +鹚 4896 +枳 4897 +嘤 4898 +愎 4899 +幔 4900 +荪 4901 +遽 4902 +馊 4903 +袂 4904 +鳟 4905 +瓠 4906 +贲 4907 +窸 4908 +挈 4909 +捋 4910 +沆 4911 +哞 4912 +椹 4913 +蓖 4914 +睚 4915 +渥 4916 +杼 4917 +棹 4918 +剽 4919 +屙 4920 +鸩 4921 +薹 4922 +珙 4923 +邙 4924 +瑁 4925 +铋 4926 +樯 4927 +滓 4928 +埭 4929 +碚 4930 +慵 4931 +膈 4932 +砧 4933 +毗 4934 +夯 4935 +徕 4936 +粳 4937 +橹 4938 +窣 4939 +獠 4940 +镊 4941 +箸 4942 +蘼 4943 +嬅 4944 +趔 4945 +吲 4946 +珅 4947 +喙 4948 +獐 4949 +璎 4950 +蛄 4951 +髌 4952 +珏 4953 +楫 4954 +倬 4955 +颏 4956 +骠 4957 +骝 4958 +悛 4959 +垭 4960 +赓 4961 +邝 4962 +秭 4963 +脘 4964 +糸 4965 +拏 4966 +堇 4967 +蛱 4968 +鎏 4969 +祎 4970 +谒 4971 +埚 4972 +怙 4973 +溆 4974 +黟 4975 +的 4976 diff --git a/athena/utils/vocabs/en.vocab b/athena/utils/vocabs/en.vocab new file mode 100644 index 00000000..fea065a6 --- /dev/null +++ b/athena/utils/vocabs/en.vocab @@ -0,0 +1,30 @@ +a 1 +b 2 +c 3 +d 4 +e 5 +f 6 +g 7 +h 8 +i 9 +j 10 +k 11 +l 12 +m 13 +n 14 +o 15 +p 16 +q 17 +r 18 +s 19 +t 20 +u 21 +v 22 +w 23 +x 24 +y 25 +z 26 +' 27 +- 28 + 0 + 29 diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..7314d23e --- /dev/null +++ b/docs/README.md @@ -0,0 +1,158 @@ + +# Athena + +*Athena* is an open-source implementation of end-to-end Automatic Speech Recognition (ASR) engine. Currently this project supports training and decoding of Connectionist Temporal Classification (CTC) based model, transformer-basesd encoder-decoder model and Hybrid CTC/attention based model, and MPC based unsupervised pretraning. + +Our vision is to empower both industrial application and academic research on end-to-end models for speech recognition. To make ASR accessible to everyone, we're also releasing some example implementation based on some opensource dataset, like HKSUT, Librispeech + +All of our models are implemented in Tensorflow>=2.0.0. + + +## Table of Contents + +- [Athena](#athena) + - [Table of Contents](#table-of-contents) + - [Key Features](#key-features) + - [Installation](#installation) + - [Data Preparation](#data-preparation) + - [Create Manifest](#create-manifest) + - [Training](#training) + - [Setting the Configuration File](#setting-the-configuration-file) + - [Train a Model](#train-a-model) + - [Results](#results) + - [Directory Structure](#directory-structure) + +## Key Features + +- Hybrid CTC/Transformer based end-to-end ASR +- Speech-Transformer +- MPC based unsupervised pretraining + +## Installation + +This project has only been tested on Python 3. We recommend creating a virtual environment and installing the python requirements there. + +```bash +git clone https://github.com/didichuxing/athena.git +pip install -r requirements.txt +python setup.py bdist_wheel sdist +python -m pip install --ignore-installed dist/athena-0.1.0*.whl +source ./tools/env.sh +``` + +## Data Preparation + +### Create Manifest + +Athena accepts a textual manifest file as data set interface, which describes speech data set in csv format. In such file, each line contains necessary meta data (e.g. key, audio path, transcription) of a speech audio. For custom data, such manifest file needs to be prepared first. An example is shown as follows: + +```csv +wav_filename wav_length_ms transcript +/dataset/train-clean-100-wav/374-180298-0000.wav 465004 chapter sixteen i might have told you of the beginning of this liaison in a few lines but i wanted you to see every step by which we came i to agree to whatever marguerite wished +/dataset/train-clean-100-wav/374-180298-0001.wav 514764 marguerite to be unable to live apart from me it was the day after the evening when she came to see me that i sent her manon lescaut from that time seeing that i could not change my mistress's life i changed my own +/dataset/train-clean-100-wav/374-180298-0002.wav 425484 i wished above all not to leave myself time to think over the position i had accepted for in spite of myself it was a great distress to me thus my life generally so calm +/dataset/train-clean-100-wav/374-180298-0003.wav 356044 assumed all at once an appearance of noise and disorder never believe however disinterested the love of a kept woman may be that it will cost one nothing +``` + +## Training + +### Setting the Configuration File + +All of our training/ inference configurations are written in config.json. Below is an example configuration file with comments to help you understand. + +```json +{ + "batch_size":32, + "num_epochs":20, + "sorta_epoch":1, + "ckpt":"examples/asr/hkust/ckpts/transformer", + + "solver_gpu":[0], + "solver_config":{ + "clip_norm":100, + "log_interval":10, + "enable_tf_function":true + }, + + "model":"speech_transformer", + "num_classes": null, + "pretrained_model": null, + "model_config":{ + "return_encoder_output":false, + "num_filters":512, + "d_model":512, + "num_heads":8, + "num_encoder_layers":12, + "num_decoder_layers":6, + "dff":1280, + "rate":0.1, + "label_smoothing_rate":0.0 + }, + + "optimizer":"warmup_adam", + "optimizer_config":{ + "d_model":512, + "warmup_steps":8000, + "k":0.5 + }, + + "dataset_builder": "speech_recognition_dataset", + "dataset_config":{ + "audio_config":{ + "type":"Fbank", + "filterbank_channel_count":40, + "local_cmvn":false + }, + "cmvn_file":"examples/asr/hkust/data/cmvn", + "vocab_file":"examples/asr/hkust/data/vocab", + "input_length_range":[10, 5000] + }, + "train_csv":"/tmp-data/dataset/opensource/hkust/train.csv", + "dev_csv":"/tmp-data/dataset/opensource/hkust/dev.csv", + "test_csv":"/tmp-data/dataset/opensource/hkust/dev.csv" +} +``` + +### Train a Model + +With all the above preparation done, training becomes straight-forward. `athena/main.py` is the entry point of the training module. Just run `python athena/main.py ` + +Please install Horovod and MPI at first, if you want to train model using multi-gpu. See the [Horovod page](https://github.com/horovod/horovod) for more instructions. + +To run on a machine with 4 GPUs with Athena: +`$ horovodrun -np 4 -H localhost:4 python athena/horovod_main.py ` + +To run on 4 machines with 4 GPUs each with Athena: +`$ horovodrun -np 16 -H server1:4,server2:4,server3:4,server4:4 python athena/horovod_main.py ` + +## Results + +Language | Model Name | Training Data | Hours of Speech | WER/% +:-----------: | :------------: | :----------: | -------: | -------: +English | Transformer | [LibriSpeech Dataset](http://www.openslr.org/12/) | 960 h | +Mandarin | Transformer | HKUST Dataset | 151 h | + +## Directory Structure + +Below is the basic directory structure for Athena + +```bash +|-- Athena +| |-- data # - root directory for input-related operations +| | |-- datasets # custom datasets for ASR and pretraining +| |-- layers # some layers +| |-- models # some models +| |-- tools # contains various tools, e.g. decoding tools +| |-- transform # custom featureizer based on C++ +| | |-- feats +| | | |-- ops # c++ code on tensorflow ops +| |-- utils # utils, e.g. checkpoit, learning_rate, metric, etc +|-- docs # docs +|-- examples # example scripts for ASR, TTS, etc +| |-- asr # each subdirectory contains a data preparation scripts and a run script for the task +| |-- aishell +| |-- hkust +| |-- librispeech +| |-- switchboard_fisher +|-- tools # need to source env.sh before training +``` diff --git a/docs/TheTrainningEfficiency.md b/docs/TheTrainningEfficiency.md new file mode 100644 index 00000000..d0024587 --- /dev/null +++ b/docs/TheTrainningEfficiency.md @@ -0,0 +1,31 @@ +# The efficiency of GPU Using '``Horovod``+``Tensorflow``' + +## Experimental + +The Training Environment: ``Athena`` + + +Traning Data: A subset was random selected 1000 samples from HKST training dataset. + + + +Newwork: ``LAS`` Model + +Primary Network Configuration: ``NUM_EPOCHS`` 1， ``BATCH_SIZE`` 10 + + + + The training time is changed by deferent number of of server and GPU when using `Horovod`+`Tensorflow`. As the same time, the training data and network structure etc still keep same to train `one` `epoch`. These results of experiment as follow: + +### The training time using ``Horovod``+``Tensorflow``(Character) + + +Server and GPU number | 1S-1GPU | 1S-2GPUs | 1S-3GPUs | 1S-4GPUs | 2Ss-2GPUs | 2Ss-4GPUs | 2Ss-6GPUs | 2Ss-8GPUs | +:-------:|:-------:|:-------:|:-------:|:-------:|:-------:|:-------:|:--------:|:--------:| +training time(s/1 epoch) | 121.409 | 83.111 | 61.607 | 54.507 | 82.486 | 49.888 | 33.333 | 28.101 | + +## The Reslut Analysis +1. As the character shown that the more GPUs are used and the training time is shorter. For example, we commpared their training time scale between using 1 server with 1 GPU and 1 server with 4 GPUs. Their training time scale is `1S-4GPUs:1S-1GPU=1:2.22`. Moreover，anoter set of data is recorded as `2Ss-8GPUs:1S-1GPU=1:4.3`. From them we can see, increasing the number of GPU when we train model can save training time and increase the efficiency. + +2. The communication time is really short between difference server using `Horovod`. We have trained the same structure model respectively using 1 servers with 2 GPUs and using 2 servers with 1 GPU each and the training time scale is `1S-2GPUs:2Ss-2GPUs=1:1`. + diff --git a/docs/development/contributing.md b/docs/development/contributing.md new file mode 100644 index 00000000..bc1f4585 --- /dev/null +++ b/docs/development/contributing.md @@ -0,0 +1,53 @@ +# Contributing Guide + +## License + +The source file should contain a license header. See the existing files as the example. + +## Name style + +All name in python and cpp using [snake case style](https://en.wikipedia.org/wiki/Snake_case), except for `op` for `Tensorflow`. +For Golang, using Camel-Case for `variable name` and `interface`. + +## Python style + +Changes to Python code should conform the [Chromium Python Style Guide](https://chromium.googlesource.com/chromium/src/+/master/styleguide/python/python.md). +You can use [yapf](https://github.com/google/yapf) to check the style. +The style configuration is `.style.yapf`. + +## C++ style + +Changes to C++ code should conform to [Google C++ Style Guide](https://github.com/google/styleguide). +You can use [cpplint](https://github.com/google/styleguide/tree/gh-pages/cpplint) to check the style and use [clang-format](https://clang.llvm.org/docs/ClangFormat.html) to format the code. +The style configuration is `.clang-format`. + +## C++ macro + +C++ macros should start with `ATHENA_`, except for most common ones like `LOG` and `VLOG`. + +## Golang style + +For Golang styple, please see docs below: + +* [How to Write Go Code](https://golang.org/doc/code.html) +* [Effective Go](https://golang.org/doc/effective_go.html#interface-names) +* [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments) +* [Golang Style in Chinese](https://juejin.im/post/5c16f16c5188252dcb30ff42) + +Before commit golang code, plase using `go fmt` and `go vec` to format and lint code. + +## Logging guideline + +For `python` using [abseil-py](https://github.com/abseil/abseil-py), [more info](https://abseil.io/docs/python/). + +For C++ using [abseil-cpp](https://github.com/abseil/abseil-cpp), [more info](https://abseil.io/docs/cpp/). + +For Golang using [glog](https://github.com/golang/glog). + +## Unit test + +For `python` using `tf.test.TestCase` + +For C++ using [googletest](https://github.com/google/googletest) + +For Golang using `go test` for unittest. diff --git a/docs/transform/speech_feature.md b/docs/transform/speech_feature.md new file mode 100644 index 00000000..8ce55e76 --- /dev/null +++ b/docs/transform/speech_feature.md @@ -0,0 +1,78 @@ +# 语音特征简介 + +## 语音数据 +在进行特征处理前，我们需要从wav格式的音频文件获取音频数据和采样率。read_wav.py提供了ReadWav的接口，使用如下： + + config = {'sample_rate': 16000.0} + wav_path = 'sm1_cln.wav'' + read_wav = ReadWav.params(config).instantiate() + audio_data, sample_rate = read_wav(wav_path) + +获得的音频数据时float类型的，范围为[-1, 1]。原始信号在时域上的表示如下图所示： +![audio data of wavfile](img/audio_data.png "Audio data") + +## 短时傅里叶变换(STFT) +语音在时域上变化是快速且比较复杂的，我们常常需要采用傅里叶变换将其转换到频域上。对数字语音信号进行的傅里叶变换即为短时傅里叶变换(Short Time Fourier Transform， STFT)。STFT的一般分为预加重(可选)、分帧、加窗、傅里叶变换这几个步骤。 +### 预加重 +由于语音信号的平均功率谱受到声门激励和口鼻辐射的影响，高频段大约在800Hz以上以6dB/倍频程跌落，所以求语音信号频谱时，频率越高相应成分越小。预加重的目的在于提高高频成分，使信号的频谱变得平坦，保持在低频到高频的整个频带中能够用同样的信噪比求频谱，以便于频谱分析。一般通过一阶FIR高通滤波器来实现预加重，其传递函数为(a为预加重系数)： + + y(n) = x(n) - ax(n-1), 0.9 < a < 1.0 +语音信号经过预加重前后的频谱对比如下图所示(a=0.97)。 +![Spectrum of orignal data](img/spectrum_orig.png "Orignal Spectrum") +![Spectrum of pre_emphasis data](img/spectrum_emph.png "Pre_emphasis Spectrum") + +### 分帧 +傅里叶变换要求信号是平稳的，但是语音在宏观上是不平稳的，所以我们首先需要将语音信号切割成较短固定长度的时间片段【即分帧】，此时语音信号可以看成是平稳的。帧长需要满足：①足够短保证帧内信号是平稳的，所以一帧的长度应小于一个音素的长度[50-200ms]；②必须包含足够多的振动周期，而语音中的基频男声为100Hz(T=10ms)，女声为200Hz(T=5ms)，所以一般至少取20ms。综上，帧长一般为20-50ms，即frame_length=0.002-0.005(单位s)。kaldi默认帧长为0.025ms，帧移位0.010ms。 +### 加窗 +对时域信号进行截断，若非周期截断就会产生频谱泄露。为了降低频谱泄露误差，我们需要使用加权函数，即窗函数，使时域信号似乎更好地满足傅里叶变换的周期性要求。每帧信号与一个平滑的窗函数相乘，可以让帧两端平滑衰减到零，这样可以降低傅里叶变换后旁瓣的强度，取得更高质量的频谱。常见的窗函数有矩形窗、汉宁窗、汉明窗等。加窗的代价是一帧信号的两端部分被削弱了，为了避免信息丢失，我们需要对其进行弥补——不要背靠背地截取而是相互重叠一部分。相邻两帧的起始位置的时间差即为帧移，一般取帧长的0-0.5。即hop_length=(0-0.5) * frame_length。 +常见的汉明窗对应的窗函数如下(N是语音总样本数，n是0-N的整数)： +![Hamming](img/hamming.png "Hamming") + +### 快速傅里叶变换(FFT) +我们获取的是数字音频，因此我们需要进行离散傅里叶变换(DFT)。其数学公式如下： + +![DFT](img/DFT.png "DFT") + +由于DFT的计算复杂度较高，我们通常使用的是快速傅里叶变换(FFT)。FFT的点数(Nfft)一般取2的整数次幂，且必须大于等于一帧信号的样本点数，如果帧长为400样本点，则Nfft必须大于等于512。帧长不等于Nfft时可以采用补零操作。频谱的分辨率精度=采样率/Nfft，第m个频点对应的频率f(m)=m × 频率分辨率。 +对于单通道语音，我们获取到的是一个复数矩阵，维度为(num_Frequencies, num_frames)，代表该语音信号是由num_Frequencies个不同相位的正弦波组合而成。对于每一个时频点，FFT的绝对值是该频率对应的正弦波的幅度，相位就是正弦波的初始相位，因此对应的频谱分别称为幅度谱和相位谱，如下图所示。 +![Amplitude Spectrum](img/amplitude_spectrum.png "Amplitude Spectrum") +![Phase Spectrum](img/phase_spectrum.png "Phase Spectrum") + +## 功率谱（Spectrum） +计算STFT后，我们得到了频域的信号。每个频带范围的能量大小不一，不同的音素的能量谱不一样。其计算公式为： +![Power_Spectrum](img/spectrum.png "Power_Spectrum") + +常用的还有log谱，即logP = 10 × log(P)。功率谱和log功率谱如下图所示。 +![Power Spectrum](img/power_spectrum.png "Power Spectrum") +![Log-power Spectrum](img/logpower_spectrum.png "Log-power Spectrum") + +## FliterBank +人耳对声音频谱的响应是非线性的，人耳对于低频声音的分辨率要高于高频的声音。经验表明：如果我们能够设计一种前端处理算法，以类似于人耳的方式对音频进行处理，可以提高语音识别的性能。FilterBank就是这样的一种算法。在功率谱的基础上获取FBank特征，需要进行Mel滤波和取对数运算。 +### Mel滤波 +通过把频率转换成Mel频率，我们的特征就能够更好的匹配人类的听觉感知效果。频率f和Mel频率m的转换公式如下： +![Mel2Freq && Freq2Mel](img/mel_freq.png "Mel2Freq && Freq2Mel") +#### Mel滤波器组 +Mel滤波器组是一组大约数量为20-40(kaldi默认为23，MFCC为26)的三角滤波器，每一个三角窗滤波器覆盖的范围都近似于人耳的一个临界带宽。三角窗口可以覆盖从0到Nyquist的整个频率范围，但是通常我们会设定频率上限和下限，屏蔽掉某些不需要或者有噪声的频率范围。Mel滤波器有两种常见的形式：中心频率响应恒为1，三角形滤波器的面积随着带宽的变化而变化；随着宽的增加而改变高度，保证其面积不变。后一种的数学表达式为： +![MelBanks_maths](img/MelBank.png "MelBanks_maths") +式中m代表第m个滤波器；k代表横轴坐标，也就是自变量；f(m)代表第m个滤波器的中心点的横坐标值。其效果图如下： +![MelBanks](img/melbanks.png "MelBanks") +Mel滤波器组有两个主要的作用：①对能量谱进行平滑化，并消除谐波的作用，突出语音的共振峰；②降低运算量。 +采用Mel滤波器组对上一步得到的功率谱估计进行滤波，得到维数和Mel滤波器组三角形个数一致的特征向量，数学表达为： +![MelFilter](img/Mel_filter.png "MelFilter") +### 对数运算 +这一步就是取上一步结果的对数，这样可以放大低能量处的能量差异。即，FBank特征为 +![FliterBank](img/logMel.png "FliterBank") +FliterBank特征的效果图如下图所示(频率上限位8000Hz，下限为20Hz，特征维数为23)： +![FliterBank Features](img/fbank.png "FliterBank Features") +## 梅尔倒谱系数(MFCC） +FBank特征已经很贴近人耳的响应特性，但是仍有一些不足：FBank特征相邻的特征高度相关（相邻滤波器组有重叠），因此当我们用HMM对音素建模的时候，几乎总需要首先进行离散余弦变换(discrete cosine transform，DCT），通过这样得到MFCC(Mel-scale FrequencyCepstral Coefficients)特征。DCT的实质是去除各维信号之间的相关性，将信号映射到低维空间。 +DCT的数学表达为： +![DCT](img/DCT.png "DCT") +N是FBank的特征维度，M 是 DCT(离散余弦变换)之后的特征维度。DCT对于一般的语音信号作用后，所得的结果的前几个系数特别大，后面的系数比较小，一次一般仅保留前12-20个，这样也进一步压缩了数据。MFCC的效果图如下： +![MFCC Features](img/MFCC.png "MFCC Features") +FBank和MFCC的对比：①FBank特征相关性较高，而DNN/CNN可以更好的利用这些相关性，使用FBank特征可以更多地降低WER；②MFCC具有更好的判别度，而对于使用对角协方差矩阵的GMM由于忽略了不同特征维度的相关性，MFCC更适合用来做特征。 +## 差分(delta） +标准的倒谱参数MFCC是针对一段语音信号进行特征提取，只反映了语音参数的静态特性，语音的动态特性可以用这些静态特征的差分谱来描述。实验证明：把动、静态特征结合起来才能有效提高系统的识别性能。差分参数的计算可以采用下面的公式(t是帧数，典型值为2)： +![Delta](img/delta.png "Delta") +## 参考： +https://haythamfayek.com/2016/04/21/speech-processing-for-machine-learning.html diff --git a/docs/transform/user_manual.md b/docs/transform/user_manual.md new file mode 100644 index 00000000..f07fab04 --- /dev/null +++ b/docs/transform/user_manual.md @@ -0,0 +1,166 @@ +# User Manual + +## Introduction + +Transform is a preprocess data toolkit. + +## Usage + +Transform support speech feature extract: + +### 1. Import module + +```python +from athena.transform import AudioFeaturizer +``` + +### 2. Init a feature extract object + +#### Read wav + +```python +conf = {'type':'ReadWav'} +feature_ext = AudioFeaturizer() + +''' +Other default args: +'audio_channels':1 + +The shape of the output: +[T, 1, 1] +''' +``` + +#### Spectrum + +```python +conf = {'type':'Spectrum'} +feature_ext = AudioFeaturizer(conf) + +''' +Other default args: +'sample_rate' : 16000 +'window_length' : 0.025 +'frame_length' : 0.010 +'global_mean': 全局均值 +'global_variance': 全局方差 +'local_cmvn' : 默认是True, 做句子cmvn + +The shape of the output: +[T, dim, 1] +''' +``` + +#### FliterBank + +```python +conf = {'type':'Fbank'} +feature_ext = AudioFeaturizer(conf) + +''' +Other default args: +'sample_rate' : 采样率 16000 +'window_length' : 窗长 0.025秒 +'frame_length' : 步长 0.010秒 +'upper_frequency_limit' : 4000 +'lower_frequency_limit': 20 +'filterbank_channel_count' : 40 +'delta_delta' : 是否做差分 False +'window' : 差分窗长 2 +'order' : 差分阶数 2 +'global_mean': 全局均值 +'global_variance': 全局方差 +'local_cmvn' : 默认是True, 做句子cmvn + +Returns: + A tensor of shape [T, dim, num_channel]. + dim = 40 + num_channel = 1 if 'delta_delta' == False else 1 + 'order' +''' +``` + +#### CMVN + +```python +conf = {'type':'CMVN'} +cmvn = AudioFeaturizer(conf) + +''' +Other configuration + +'global_mean': global cmvn +'global_variance': global variance +'local_cmvn' : default true + +'global_mean'和'global_variance'如果设置则会做全局cmvn，否则不做全局cmvn。 +'local_cmvn' 设置False不做句子cmvn +''' +``` + +### 3. Feature extract + +```python +feat = feature_ext(audio) +''' +audio : Audio file or audio data. +feat : A tensor containing speech feature. +''' +``` + +### 4. Get feature dim and the number of channels + +```python +dim = feature_ext.dim +# dim : A int scalar, it is feature dim. +num_channels = feature_ext.num_channels +# num_channels: A int scalar, it is the number of channels of the output feature +# the shape of the output features: [None, dim, num_channels] +``` + +### 5. Example + +#### 5.1 Extract speech feature filterbank from audio file: + +```python +import tensorflow as tf +from transform.audio_featurizer import AudioFeaturizer + +audio_file = 'englist.wav' +conf = {'type':'Fbank' + 'sample_rate':8000, + 'delta_delta': True + } +feature_ext = AudioFeaturizer(conf) +dim = feature_ext.dim +print('Dim size is ', dim) +num_channels = feature_ext.num_channels +print('num_channels is ', num_channels) + +feat = feature_ext(audio_file) +with tf.Session() as sess: + fbank = sess.run(feat) + print('Fbank shape is ', fbank.shape) +``` + +Result: + +``` +Dim is 40 +Fbank shape is (346, 40, 3) # [T, D, C] +``` + +#### 5.2 CMVN usage: + +```python +import tensorflow as tf +from transform.audio_featurizer import AudioFeaturizer + +dim = 23 +config = {'type': 'CMVN', + 'global_mean': np.zeros(dim).tolist(), + 'global_variance': np.ones(dim).tolist(), + 'local_cmvn': True} +cmvn = AudioFeaturizer(config) +audio_feature = tf.compat.v1.random_uniform(shape=[10, dim], dtype=tf.float32, maxval=1.0) +print('cmvn : ', cmvn(audio_feature)) +``` diff --git a/examples/asr/README.md b/examples/asr/README.md new file mode 100644 index 00000000..3e2fd4ed --- /dev/null +++ b/examples/asr/README.md @@ -0,0 +1,22 @@ + +# Examples for HKUST + +## 1 Transformer + +```bash +source env.sh +python examples/asr/hkust/prepare_data.py /tmp-data/dataset/opensource/hkust +python athena/main.py examples/asr/hkust/transformer.json +``` + +## 2 MTL_Transformer_CTC + +```bash +source env.sh +python examples/asr/hkust/prepare_data.py /tmp-data/dataset/opensource/hkust +python athena/main.py examples/asr/hkust/mtl_transformer.json +``` + +# Examples for Librispeech + +TODO: need test diff --git a/examples/asr/aishell/data/vocab b/examples/asr/aishell/data/vocab new file mode 100644 index 00000000..dd9c101d --- /dev/null +++ b/examples/asr/aishell/data/vocab @@ -0,0 +1,4232 @@ + 1 + 2 +一 3 +丁 4 +七 5 +万 6 +丈 7 +三 8 +上 9 +下 10 +不 11 +与 12 +丐 13 +丑 14 +专 15 +且 16 +世 17 +丘 18 +丙 19 +业 20 +丛 21 +东 22 +丝 23 +丞 24 +丢 25 +两 26 +严 27 +丧 28 +个 29 +丫 30 +中 31 +丰 32 +串 33 +临 34 +丸 35 +丹 36 +为 37 +主 38 +丽 39 +举 40 +乃 41 +久 42 +么 43 +义 44 +之 45 +乌 46 +乍 47 +乎 48 +乏 49 +乐 50 +乒 51 +乓 52 +乔 53 +乖 54 +乘 55 +乙 56 +九 57 +乞 58 +也 59 +习 60 +乡 61 +书 62 +买 63 +乱 64 +乳 65 +乾 66 +了 67 +予 68 +争 69 +事 70 +二 71 +于 72 +亏 73 +云 74 +互 75 +五 76 +井 77 +亚 78 +些 79 +亟 80 +亡 81 +亢 82 +交 83 +亥 84 +亦 85 +产 86 +亨 87 +亩 88 +享 89 +京 90 +亭 91 +亮 92 +亲 93 +亳 94 +亵 95 +人 96 +亿 97 +什 98 +仁 99 +仄 100 +仅 101 +仇 102 +今 103 +介 104 +仍 105 +从 106 +仑 107 +仓 108 +仔 109 +仕 110 +他 111 +仗 112 +付 113 +仙 114 +仡 115 +代 116 +令 117 +以 118 +仨 119 +仪 120 +们 121 +仰 122 +仲 123 +件 124 +价 125 +任 126 +份 127 +仿 128 +企 129 +伉 130 +伊 131 +伍 132 +伎 133 +伏 134 +伐 135 +休 136 +众 137 +优 138 +伙 139 +会 140 +伞 141 +伟 142 +传 143 +伢 144 +伤 145 +伦 146 +伪 147 +伯 148 +估 149 +伴 150 +伶 151 +伸 152 +伺 153 +似 154 +伽 155 +佃 156 +但 157 +位 158 +低 159 +住 160 +佐 161 +佑 162 +体 163 +何 164 +佘 165 +余 166 +佛 167 +作 168 +佟 169 +你 170 +佣 171 +佩 172 +佬 173 +佳 174 +佶 175 +佼 176 +使 177 +侃 178 +侄 179 +侈 180 +例 181 +侍 182 +侑 183 +侗 184 +供 185 +依 186 +侠 187 +侣 188 +侥 189 +侦 190 +侧 191 +侨 192 +侬 193 +侮 194 +侯 195 +侵 196 +便 197 +促 198 +俄 199 +俊 200 +俏 201 +俐 202 +俗 203 +俘 204 +俚 205 +保 206 +俞 207 +信 208 +俨 209 +俩 210 +俪 211 +俭 212 +修 213 +俯 214 +俱 215 +俸 216 +俺 217 +俾 218 +倍 219 +倒 220 +倘 221 +候 222 +倚 223 +倜 224 +借 225 +倡 226 +倦 227 +倩 228 +倪 229 +债 230 +值 231 +倾 232 +假 233 +偏 234 +做 235 +停 236 +健 237 +偶 238 +偷 239 +偿 240 +傅 241 +傍 242 +傥 243 +储 244 +催 245 +傲 246 +傻 247 +像 248 +僚 249 +僧 250 +僮 251 +僵 252 +僻 253 +儒 254 +儿 255 +兀 256 +允 257 +元 258 +兄 259 +充 260 +兆 261 +先 262 +光 263 +克 264 +免 265 +兑 266 +兔 267 +兖 268 +党 269 +兜 270 +兢 271 +入 272 +全 273 +八 274 +公 275 +六 276 +兰 277 +共 278 +关 279 +兴 280 +兵 281 +其 282 +具 283 +典 284 +兹 285 +养 286 +兼 287 +兽 288 +冀 289 +内 290 +冈 291 +冉 292 +册 293 +再 294 +冒 295 +冕 296 +写 297 +军 298 +农 299 +冠 300 +冤 301 +冥 302 +冬 303 +冯 304 +冰 305 +冲 306 +决 307 +况 308 +冶 309 +冷 310 +冻 311 +净 312 +凄 313 +准 314 +凇 315 +凉 316 +凋 317 +凌 318 +减 319 +凑 320 +凝 321 +几 322 +凡 323 +凤 324 +凭 325 +凯 326 +凰 327 +凳 328 +凶 329 +凸 330 +凹 331 +出 332 +击 333 +函 334 +凿 335 +刀 336 +刁 337 +刃 338 +分 339 +切 340 +刊 341 +刑 342 +划 343 +列 344 +刘 345 +则 346 +刚 347 +创 348 +初 349 +删 350 +判 351 +刨 352 +利 353 +别 354 +刮 355 +到 356 +制 357 +刷 358 +券 359 +刹 360 +刺 361 +刻 362 +剁 363 +剂 364 +剃 365 +削 366 +前 367 +剐 368 +剑 369 +剔 370 +剖 371 +剥 372 +剧 373 +剩 374 +剪 375 +副 376 +割 377 +剽 378 +剿 379 +劈 380 +力 381 +劝 382 +办 383 +功 384 +加 385 +务 386 +劣 387 +动 388 +助 389 +努 390 +劫 391 +励 392 +劲 393 +劳 394 +劵 395 +势 396 +勃 397 +勇 398 +勉 399 +勋 400 +勒 401 +勘 402 +募 403 +勤 404 +勺 405 +勾 406 +勿 407 +匀 408 +包 409 +匆 410 +匈 411 +匕 412 +化 413 +北 414 +匙 415 +匝 416 +匠 417 +匡 418 +匣 419 +匪 420 +匮 421 +匹 422 +区 423 +医 424 +匾 425 +匿 426 +十 427 +千 428 +升 429 +午 430 +卉 431 +半 432 +华 433 +协 434 +卑 435 +卒 436 +卓 437 +单 438 +卖 439 +南 440 +博 441 +卜 442 +卞 443 +占 444 +卡 445 +卢 446 +卤 447 +卦 448 +卧 449 +卫 450 +卯 451 +印 452 +危 453 +卲 454 +即 455 +却 456 +卵 457 +卷 458 +卸 459 +卿 460 +厂 461 +厄 462 +厅 463 +历 464 +厉 465 +压 466 +厌 467 +厕 468 +厘 469 +厚 470 +原 471 +厢 472 +厥 473 +厦 474 +厨 475 +厩 476 +厮 477 +去 478 +县 479 +参 480 +又 481 +叉 482 +及 483 +友 484 +双 485 +反 486 +发 487 +叔 488 +取 489 +受 490 +变 491 +叙 492 +叛 493 +叠 494 +口 495 +古 496 +句 497 +另 498 +叨 499 +叩 500 +只 501 +叫 502 +召 503 +叭 504 +叮 505 +可 506 +台 507 +叱 508 +史 509 +右 510 +叵 511 +叶 512 +号 513 +司 514 +叹 515 +叼 516 +吁 517 +吃 518 +各 519 +吆 520 +合 521 +吉 522 +吊 523 +同 524 +名 525 +后 526 +吏 527 +吐 528 +向 529 +吓 530 +吕 531 +吗 532 +君 533 +吝 534 +吞 535 +吟 536 +否 537 +吧 538 +吨 539 +吩 540 +含 541 +听 542 +吭 543 +启 544 +吴 545 +吵 546 +吸 547 +吹 548 +吻 549 +吼 550 +吾 551 +吿 552 +呀 553 +呃 554 +呆 555 +呈 556 +告 557 +呐 558 +呕 559 +呗 560 +员 561 +呛 562 +呜 563 +呢 564 +呦 565 +周 566 +呲 567 +味 568 +呵 569 +呼 570 +命 571 +咀 572 +咄 573 +咋 574 +和 575 +咎 576 +咏 577 +咐 578 +咒 579 +咔 580 +咕 581 +咖 582 +咚 583 +咣 584 +咤 585 +咧 586 +咨 587 +咪 588 +咫 589 +咬 590 +咯 591 +咱 592 +咳 593 +咸 594 +咽 595 +哀 596 +品 597 +哄 598 +哆 599 +哇 600 +哈 601 +哉 602 +响 603 +哎 604 +哑 605 +哒 606 +哗 607 +哟 608 +哥 609 +哦 610 +哨 611 +哪 612 +哭 613 +哲 614 +哺 615 +哼 616 +哽 617 +唁 618 +唇 619 +唉 620 +唏 621 +唐 622 +唠 623 +唤 624 +唬 625 +售 626 +唯 627 +唱 628 +唾 629 +啃 630 +商 631 +啊 632 +啕 633 +啡 634 +啤 635 +啥 636 +啦 637 +啧 638 +啪 639 +啬 640 +啰 641 +啵 642 +啶 643 +啸 644 +啼 645 +喀 646 +喂 647 +善 648 +喆 649 +喇 650 +喉 651 +喊 652 +喔 653 +喘 654 +喜 655 +喝 656 +喧 657 +喱 658 +喵 659 +喷 660 +喻 661 +喽 662 +嗅 663 +嗑 664 +嗒 665 +嗓 666 +嗡 667 +嗣 668 +嗤 669 +嗦 670 +嗨 671 +嗬 672 +嗯 673 +嗲 674 +嗷 675 +嗽 676 +嘀 677 +嘉 678 +嘎 679 +嘘 680 +嘛 681 +嘟 682 +嘭 683 +嘱 684 +嘲 685 +嘴 686 +嘻 687 +噎 688 +器 689 +噩 690 +噪 691 +噬 692 +噱 693 +噼 694 +嚎 695 +嚏 696 +嚓 697 +嚣 698 +嚷 699 +嚼 700 +囊 701 +囚 702 +四 703 +回 704 +因 705 +团 706 +囤 707 +囧 708 +园 709 +困 710 +围 711 +固 712 +国 713 +图 714 +圆 715 +圈 716 +土 717 +圣 718 +在 719 +圩 720 +圪 721 +圭 722 +地 723 +圳 724 +场 725 +圾 726 +址 727 +坂 728 +均 729 +坊 730 +坍 731 +坎 732 +坏 733 +坐 734 +坑 735 +块 736 +坚 737 +坛 738 +坝 739 +坞 740 +坟 741 +坠 742 +坡 743 +坤 744 +坦 745 +坪 746 +坯 747 +坷 748 +垂 749 +垃 750 +垄 751 +垅 752 +型 753 +垌 754 +垒 755 +垛 756 +垢 757 +垣 758 +垤 759 +垦 760 +垫 761 +垮 762 +埃 763 +埋 764 +城 765 +埔 766 +埜 767 +域 768 +培 769 +基 770 +堂 771 +堆 772 +堕 773 +堡 774 +堤 775 +堪 776 +堰 777 +堵 778 +塌 779 +塑 780 +塔 781 +塘 782 +塞 783 +填 784 +塬 785 +塾 786 +境 787 +墅 788 +墓 789 +墙 790 +增 791 +墟 792 +墨 793 +墩 794 +壁 795 +壑 796 +壕 797 +壤 798 +士 799 +壮 800 +声 801 +壳 802 +壶 803 +壹 804 +处 805 +备 806 +复 807 +夏 808 +夕 809 +外 810 +夙 811 +多 812 +夜 813 +够 814 +大 815 +天 816 +太 817 +夫 818 +夭 819 +央 820 +夯 821 +失 822 +头 823 +夷 824 +夸 825 +夹 826 +夺 827 +奂 828 +奇 829 +奈 830 +奉 831 +奋 832 +奎 833 +奏 834 +契 835 +奔 836 +奕 837 +奖 838 +套 839 +奘 840 +奚 841 +奠 842 +奢 843 +奥 844 +女 845 +奴 846 +奶 847 +奸 848 +她 849 +好 850 +如 851 +妃 852 +妄 853 +妆 854 +妇 855 +妈 856 +妊 857 +妍 858 +妒 859 +妖 860 +妙 861 +妞 862 +妤 863 +妥 864 +妧 865 +妨 866 +妩 867 +妮 868 +妯 869 +妹 870 +妻 871 +姆 872 +姊 873 +始 874 +姐 875 +姑 876 +姓 877 +委 878 +姗 879 +姚 880 +姜 881 +姝 882 +姣 883 +姥 884 +姨 885 +姬 886 +姻 887 +姿 888 +威 889 +娃 890 +娄 891 +娅 892 +娇 893 +娌 894 +娘 895 +娜 896 +娟 897 +娠 898 +娥 899 +娩 900 +娱 901 +娴 902 +娶 903 +娼 904 +婀 905 +婆 906 +婉 907 +婕 908 +婚 909 +婧 910 +婪 911 +婴 912 +婵 913 +婶 914 +婷 915 +婿 916 +媒 917 +媚 918 +媛 919 +媞 920 +媲 921 +媳 922 +嫁 923 +嫂 924 +嫉 925 +嫌 926 +嫔 927 +嫖 928 +嫚 929 +嫣 930 +嫦 931 +嫩 932 +嬉 933 +嬛 934 +嬷 935 +孀 936 +子 937 +孔 938 +孕 939 +字 940 +存 941 +孙 942 +孚 943 +孜 944 +孝 945 +孟 946 +孢 947 +季 948 +孤 949 +学 950 +孩 951 +孪 952 +孰 953 +孱 954 +孵 955 +孺 956 +宁 957 +它 958 +宅 959 +宇 960 +守 961 +安 962 +宋 963 +完 964 +宏 965 +宓 966 +宕 967 +宗 968 +官 969 +宙 970 +定 971 +宛 972 +宜 973 +宝 974 +实 975 +宠 976 +审 977 +客 978 +宣 979 +室 980 +宦 981 +宪 982 +宫 983 +宰 984 +害 985 +宴 986 +宵 987 +家 988 +宸 989 +容 990 +宽 991 +宾 992 +宿 993 +寂 994 +寄 995 +寅 996 +密 997 +寇 998 +富 999 +寐 1000 +寒 1001 +寓 1002 +寝 1003 +寞 1004 +察 1005 +寡 1006 +寥 1007 +寨 1008 +寮 1009 +寰 1010 +寸 1011 +对 1012 +寺 1013 +寻 1014 +导 1015 +寿 1016 +封 1017 +射 1018 +将 1019 +尊 1020 +小 1021 +少 1022 +尔 1023 +尖 1024 +尘 1025 +尚 1026 +尝 1027 +尤 1028 +尧 1029 +尬 1030 +就 1031 +尴 1032 +尸 1033 +尹 1034 +尺 1035 +尼 1036 +尽 1037 +尾 1038 +尿 1039 +局 1040 +屁 1041 +层 1042 +居 1043 +屈 1044 +届 1045 +屋 1046 +屌 1047 +屎 1048 +屏 1049 +屑 1050 +展 1051 +属 1052 +屠 1053 +屡 1054 +履 1055 +屯 1056 +山 1057 +屹 1058 +屿 1059 +岁 1060 +岂 1061 +岌 1062 +岐 1063 +岔 1064 +岖 1065 +岗 1066 +岚 1067 +岛 1068 +岩 1069 +岬 1070 +岭 1071 +岱 1072 +岳 1073 +岷 1074 +岸 1075 +峁 1076 +峙 1077 +峡 1078 +峥 1079 +峨 1080 +峪 1081 +峭 1082 +峰 1083 +峻 1084 +崂 1085 +崃 1086 +崇 1087 +崎 1088 +崔 1089 +崖 1090 +崛 1091 +崧 1092 +崩 1093 +崭 1094 +崴 1095 +嵋 1096 +嵌 1097 +嵘 1098 +嵛 1099 +嵩 1100 +嶝 1101 +巅 1102 +巍 1103 +川 1104 +州 1105 +巡 1106 +巢 1107 +工 1108 +左 1109 +巧 1110 +巨 1111 +巩 1112 +巫 1113 +差 1114 +己 1115 +已 1116 +巴 1117 +巷 1118 +巾 1119 +巿 1120 +币 1121 +市 1122 +布 1123 +帅 1124 +帆 1125 +师 1126 +希 1127 +帐 1128 +帕 1129 +帖 1130 +帘 1131 +帚 1132 +帜 1133 +帝 1134 +带 1135 +席 1136 +帮 1137 +帷 1138 +常 1139 +帼 1140 +帽 1141 +幂 1142 +幄 1143 +幅 1144 +幌 1145 +幕 1146 +幢 1147 +干 1148 +平 1149 +年 1150 +并 1151 +幸 1152 +幺 1153 +幻 1154 +幼 1155 +幽 1156 +广 1157 +庄 1158 +庆 1159 +庇 1160 +床 1161 +序 1162 +庐 1163 +库 1164 +应 1165 +底 1166 +店 1167 +庙 1168 +庚 1169 +府 1170 +庞 1171 +废 1172 +度 1173 +座 1174 +庭 1175 +庵 1176 +康 1177 +庸 1178 +庾 1179 +廉 1180 +廊 1181 +廓 1182 +廖 1183 +延 1184 +廷 1185 +建 1186 +开 1187 +异 1188 +弃 1189 +弄 1190 +弈 1191 +弊 1192 +式 1193 +弓 1194 +引 1195 +弗 1196 +弘 1197 +弛 1198 +弟 1199 +张 1200 +弥 1201 +弦 1202 +弧 1203 +弩 1204 +弯 1205 +弱 1206 +弹 1207 +强 1208 +归 1209 +当 1210 +录 1211 +彝 1212 +形 1213 +彤 1214 +彦 1215 +彩 1216 +彪 1217 +彬 1218 +彭 1219 +彰 1220 +影 1221 +彷 1222 +役 1223 +彻 1224 +彼 1225 +彿 1226 +往 1227 +征 1228 +径 1229 +待 1230 +徇 1231 +很 1232 +徉 1233 +徊 1234 +律 1235 +徐 1236 +徒 1237 +得 1238 +徘 1239 +徙 1240 +御 1241 +循 1242 +微 1243 +德 1244 +徽 1245 +心 1246 +必 1247 +忆 1248 +忌 1249 +忍 1250 +忐 1251 +忑 1252 +志 1253 +忘 1254 +忙 1255 +忠 1256 +忧 1257 +忪 1258 +快 1259 +忱 1260 +念 1261 +忽 1262 +怀 1263 +态 1264 +怂 1265 +怎 1266 +怒 1267 +怕 1268 +怖 1269 +怜 1270 +思 1271 +怠 1272 +怡 1273 +急 1274 +怦 1275 +性 1276 +怨 1277 +怪 1278 +怯 1279 +怵 1280 +总 1281 +恋 1282 +恍 1283 +恐 1284 +恒 1285 +恙 1286 +恢 1287 +恣 1288 +恤 1289 +恨 1290 +恩 1291 +恪 1292 +恬 1293 +恭 1294 +息 1295 +恰 1296 +恳 1297 +恶 1298 +恸 1299 +恺 1300 +恼 1301 +恿 1302 +悄 1303 +悉 1304 +悍 1305 +悔 1306 +悖 1307 +悚 1308 +悟 1309 +悠 1310 +患 1311 +悦 1312 +您 1313 +悬 1314 +悯 1315 +悲 1316 +悴 1317 +悸 1318 +悼 1319 +情 1320 +惊 1321 +惋 1322 +惑 1323 +惕 1324 +惚 1325 +惜 1326 +惟 1327 +惠 1328 +惦 1329 +惧 1330 +惨 1331 +惩 1332 +惫 1333 +惬 1334 +惮 1335 +惯 1336 +惰 1337 +想 1338 +惶 1339 +惹 1340 +惺 1341 +愁 1342 +愈 1343 +愉 1344 +意 1345 +愕 1346 +愚 1347 +感 1348 +愤 1349 +愧 1350 +愿 1351 +慈 1352 +慌 1353 +慎 1354 +慑 1355 +慕 1356 +慢 1357 +慧 1358 +慨 1359 +慰 1360 +慷 1361 +憋 1362 +憔 1363 +憧 1364 +憨 1365 +憩 1366 +憬 1367 +憷 1368 +憾 1369 +懂 1370 +懈 1371 +懊 1372 +懋 1373 +懒 1374 +懵 1375 +懿 1376 +戈 1377 +戎 1378 +戏 1379 +成 1380 +我 1381 +戒 1382 +或 1383 +战 1384 +戚 1385 +戛 1386 +戟 1387 +截 1388 +戬 1389 +戮 1390 +戳 1391 +戴 1392 +户 1393 +房 1394 +所 1395 +扁 1396 +扇 1397 +扉 1398 +手 1399 +才 1400 +扎 1401 +扑 1402 +扒 1403 +打 1404 +扔 1405 +托 1406 +扛 1407 +扣 1408 +执 1409 +扩 1410 +扫 1411 +扬 1412 +扭 1413 +扮 1414 +扯 1415 +扰 1416 +扳 1417 +扶 1418 +批 1419 +扼 1420 +找 1421 +承 1422 +技 1423 +抄 1424 +抉 1425 +把 1426 +抑 1427 +抒 1428 +抓 1429 +投 1430 +抖 1431 +抗 1432 +折 1433 +抚 1434 +抛 1435 +抠 1436 +抡 1437 +抢 1438 +护 1439 +报 1440 +抨 1441 +披 1442 +抬 1443 +抱 1444 +抵 1445 +抹 1446 +押 1447 +抽 1448 +抿 1449 +拄 1450 +担 1451 +拆 1452 +拇 1453 +拈 1454 +拉 1455 +拌 1456 +拍 1457 +拎 1458 +拐 1459 +拒 1460 +拓 1461 +拔 1462 +拖 1463 +拗 1464 +拘 1465 +拙 1466 +招 1467 +拜 1468 +拟 1469 +拢 1470 +拣 1471 +拥 1472 +拦 1473 +拧 1474 +拨 1475 +择 1476 +括 1477 +拭 1478 +拮 1479 +拯 1480 +拱 1481 +拳 1482 +拴 1483 +拷 1484 +拼 1485 +拽 1486 +拾 1487 +拿 1488 +持 1489 +挂 1490 +指 1491 +按 1492 +挎 1493 +挑 1494 +挖 1495 +挚 1496 +挛 1497 +挝 1498 +挟 1499 +挠 1500 +挡 1501 +挣 1502 +挤 1503 +挥 1504 +挨 1505 +挪 1506 +挫 1507 +振 1508 +挺 1509 +挽 1510 +捂 1511 +捅 1512 +捆 1513 +捉 1514 +捍 1515 +捎 1516 +捏 1517 +捐 1518 +捕 1519 +捞 1520 +损 1521 +捡 1522 +换 1523 +捣 1524 +捧 1525 +据 1526 +捷 1527 +捺 1528 +捻 1529 +掀 1530 +掂 1531 +授 1532 +掉 1533 +掌 1534 +掏 1535 +掐 1536 +排 1537 +掖 1538 +掘 1539 +掠 1540 +探 1541 +掣 1542 +接 1543 +控 1544 +推 1545 +掩 1546 +措 1547 +掬 1548 +掮 1549 +掰 1550 +掴 1551 +掷 1552 +掺 1553 +揉 1554 +揍 1555 +描 1556 +提 1557 +插 1558 +握 1559 +揣 1560 +揩 1561 +揪 1562 +揭 1563 +援 1564 +揽 1565 +搀 1566 +搁 1567 +搂 1568 +搅 1569 +搏 1570 +搜 1571 +搞 1572 +搡 1573 +搪 1574 +搬 1575 +搭 1576 +携 1577 +搽 1578 +摁 1579 +摄 1580 +摆 1581 +摇 1582 +摊 1583 +摒 1584 +摔 1585 +摘 1586 +摧 1587 +摩 1588 +摸 1589 +摹 1590 +撂 1591 +撇 1592 +撑 1593 +撒 1594 +撕 1595 +撞 1596 +撤 1597 +撩 1598 +撬 1599 +播 1600 +撮 1601 +撰 1602 +撵 1603 +撸 1604 +撼 1605 +擂 1606 +擅 1607 +操 1608 +擎 1609 +擒 1610 +擘 1611 +擞 1612 +擦 1613 +攀 1614 +攒 1615 +攥 1616 +支 1617 +收 1618 +改 1619 +攻 1620 +放 1621 +政 1622 +故 1623 +效 1624 +敌 1625 +敏 1626 +救 1627 +敖 1628 +教 1629 +敛 1630 +敝 1631 +敞 1632 +敢 1633 +散 1634 +敦 1635 +敬 1636 +数 1637 +敲 1638 +整 1639 +敷 1640 +文 1641 +斌 1642 +斐 1643 +斑 1644 +斓 1645 +斗 1646 +料 1647 +斛 1648 +斜 1649 +斟 1650 +斤 1651 +斥 1652 +斧 1653 +斩 1654 +断 1655 +斯 1656 +新 1657 +方 1658 +施 1659 +旁 1660 +旅 1661 +旋 1662 +族 1663 +旗 1664 +无 1665 +既 1666 +日 1667 +旦 1668 +旧 1669 +旨 1670 +早 1671 +旬 1672 +旭 1673 +旱 1674 +时 1675 +旷 1676 +旺 1677 +昀 1678 +昂 1679 +昆 1680 +昊 1681 +昌 1682 +明 1683 +昏 1684 +易 1685 +昔 1686 +昕 1687 +昙 1688 +星 1689 +映 1690 +春 1691 +昧 1692 +昨 1693 +昭 1694 +是 1695 +昱 1696 +昵 1697 +昼 1698 +显 1699 +晃 1700 +晋 1701 +晏 1702 +晒 1703 +晓 1704 +晔 1705 +晕 1706 +晖 1707 +晗 1708 +晚 1709 +晟 1710 +晤 1711 +晦 1712 +晨 1713 +普 1714 +景 1715 +晰 1716 +晴 1717 +晶 1718 +智 1719 +晾 1720 +暂 1721 +暄 1722 +暇 1723 +暑 1724 +暖 1725 +暗 1726 +暧 1727 +暨 1728 +暮 1729 +暴 1730 +曙 1731 +曝 1732 +曦 1733 +曰 1734 +曲 1735 +更 1736 +曹 1737 +曼 1738 +曾 1739 +替 1740 +最 1741 +月 1742 +有 1743 +朋 1744 +服 1745 +朐 1746 +朔 1747 +朗 1748 +望 1749 +朝 1750 +期 1751 +朦 1752 +木 1753 +未 1754 +末 1755 +本 1756 +札 1757 +术 1758 +朱 1759 +朴 1760 +朵 1761 +机 1762 +朽 1763 +杀 1764 +杂 1765 +权 1766 +杆 1767 +杉 1768 +李 1769 +杏 1770 +材 1771 +村 1772 +杖 1773 +杜 1774 +杞 1775 +束 1776 +杠 1777 +条 1778 +来 1779 +杨 1780 +杭 1781 +杯 1782 +杰 1783 +杳 1784 +松 1785 +板 1786 +极 1787 +构 1788 +枉 1789 +析 1790 +枕 1791 +林 1792 +枚 1793 +果 1794 +枝 1795 +枞 1796 +枢 1797 +枣 1798 +枪 1799 +枫 1800 +枭 1801 +枯 1802 +架 1803 +枷 1804 +柄 1805 +柏 1806 +某 1807 +染 1808 +柔 1809 +柜 1810 +柞 1811 +柠 1812 +查 1813 +柬 1814 +柯 1815 +柱 1816 +柳 1817 +柴 1818 +柿 1819 +栅 1820 +标 1821 +栈 1822 +栋 1823 +栏 1824 +树 1825 +栓 1826 +栖 1827 +栗 1828 +校 1829 +株 1830 +样 1831 +核 1832 +根 1833 +格 1834 +栽 1835 +栾 1836 +桂 1837 +桃 1838 +框 1839 +案 1840 +桉 1841 +桌 1842 +桎 1843 +桐 1844 +桑 1845 +桓 1846 +桔 1847 +档 1848 +桥 1849 +桦 1850 +桩 1851 +桶 1852 +梁 1853 +梅 1854 +梓 1855 +梗 1856 +梦 1857 +梧 1858 +梨 1859 +梭 1860 +梯 1861 +械 1862 +梳 1863 +梵 1864 +检 1865 +棉 1866 +棋 1867 +棍 1868 +棒 1869 +棕 1870 +棘 1871 +棚 1872 +棠 1873 +森 1874 +棱 1875 +棵 1876 +棺 1877 +椅 1878 +椋 1879 +植 1880 +椎 1881 +椒 1882 +椰 1883 +椿 1884 +楂 1885 +楔 1886 +楚 1887 +楞 1888 +楠 1889 +楣 1890 +楷 1891 +楼 1892 +概 1893 +榄 1894 +榆 1895 +榈 1896 +榉 1897 +榔 1898 +榕 1899 +榜 1900 +榨 1901 +榭 1902 +榴 1903 +榷 1904 +榻 1905 +槌 1906 +槎 1907 +槐 1908 +槛 1909 +槟 1910 +槽 1911 +槿 1912 +樊 1913 +樟 1914 +模 1915 +横 1916 +樱 1917 +橄 1918 +橘 1919 +橙 1920 +橡 1921 +橱 1922 +檀 1923 +檐 1924 +檬 1925 +欠 1926 +次 1927 +欢 1928 +欣 1929 +欧 1930 +欲 1931 +欺 1932 +款 1933 +歆 1934 +歇 1935 +歉 1936 +歌 1937 +止 1938 +正 1939 +此 1940 +步 1941 +武 1942 +歧 1943 +歪 1944 +歹 1945 +死 1946 +殃 1947 +殆 1948 +殉 1949 +殊 1950 +残 1951 +殒 1952 +殓 1953 +殖 1954 +殚 1955 +殡 1956 +殭 1957 +殴 1958 +段 1959 +殷 1960 +殿 1961 +毁 1962 +毂 1963 +毅 1964 +毋 1965 +母 1966 +每 1967 +毒 1968 +毓 1969 +比 1970 +毕 1971 +毗 1972 +毙 1973 +毛 1974 +毫 1975 +毯 1976 +毽 1977 +氏 1978 +民 1979 +氓 1980 +气 1981 +氛 1982 +氟 1983 +氢 1984 +氦 1985 +氧 1986 +氨 1987 +氪 1988 +氮 1989 +氯 1990 +氰 1991 +水 1992 +永 1993 +汀 1994 +汁 1995 +求 1996 +汇 1997 +汉 1998 +汕 1999 +汗 2000 +汛 2001 +汝 2002 +汞 2003 +江 2004 +池 2005 +污 2006 +汤 2007 +汪 2008 +汰 2009 +汲 2010 +汴 2011 +汶 2012 +汹 2013 +汽 2014 +汾 2015 +沁 2016 +沃 2017 +沅 2018 +沈 2019 +沉 2020 +沏 2021 +沐 2022 +沓 2023 +沙 2024 +沛 2025 +沟 2026 +没 2027 +沣 2028 +沥 2029 +沦 2030 +沧 2031 +沪 2032 +沫 2033 +沮 2034 +沱 2035 +河 2036 +沸 2037 +油 2038 +治 2039 +沼 2040 +沽 2041 +沾 2042 +沿 2043 +泄 2044 +泉 2045 +泊 2046 +泌 2047 +泓 2048 +泔 2049 +法 2050 +泗 2051 +泛 2052 +泞 2053 +泠 2054 +泡 2055 +波 2056 +泣 2057 +泥 2058 +注 2059 +泪 2060 +泯 2061 +泰 2062 +泱 2063 +泳 2064 +泵 2065 +泷 2066 +泸 2067 +泻 2068 +泼 2069 +泽 2070 +泾 2071 +洁 2072 +洋 2073 +洒 2074 +洗 2075 +洙 2076 +洛 2077 +洞 2078 +津 2079 +洪 2080 +洱 2081 +洲 2082 +洵 2083 +活 2084 +洼 2085 +洽 2086 +派 2087 +流 2088 +浅 2089 +浆 2090 +浇 2091 +浈 2092 +浊 2093 +测 2094 +济 2095 +浏 2096 +浑 2097 +浓 2098 +浙 2099 +浚 2100 +浦 2101 +浩 2102 +浪 2103 +浮 2104 +浴 2105 +海 2106 +浸 2107 +涂 2108 +涅 2109 +消 2110 +涉 2111 +涌 2112 +涎 2113 +涓 2114 +涕 2115 +涛 2116 +涝 2117 +涞 2118 +涠 2119 +涡 2120 +涤 2121 +润 2122 +涧 2123 +涨 2124 +涩 2125 +涮 2126 +涯 2127 +液 2128 +涵 2129 +涿 2130 +淀 2131 +淄 2132 +淆 2133 +淇 2134 +淋 2135 +淌 2136 +淑 2137 +淖 2138 +淘 2139 +淝 2140 +淞 2141 +淡 2142 +淤 2143 +淫 2144 +淮 2145 +深 2146 +淳 2147 +混 2148 +淹 2149 +添 2150 +淼 2151 +渀 2152 +清 2153 +渊 2154 +渍 2155 +渎 2156 +渐 2157 +渔 2158 +渗 2159 +渚 2160 +渝 2161 +渠 2162 +渡 2163 +渣 2164 +渤 2165 +渥 2166 +温 2167 +渭 2168 +港 2169 +渲 2170 +渴 2171 +游 2172 +渺 2173 +湃 2174 +湍 2175 +湖 2176 +湘 2177 +湛 2178 +湾 2179 +湿 2180 +溃 2181 +溅 2182 +溉 2183 +源 2184 +溜 2185 +溢 2186 +溥 2187 +溧 2188 +溪 2189 +溯 2190 +溶 2191 +溺 2192 +滁 2193 +滇 2194 +滋 2195 +滑 2196 +滔 2197 +滕 2198 +滚 2199 +滞 2200 +满 2201 +滢 2202 +滤 2203 +滥 2204 +滨 2205 +滩 2206 +滴 2207 +漂 2208 +漆 2209 +漏 2210 +漓 2211 +演 2212 +漕 2213 +漠 2214 +漩 2215 +漫 2216 +漭 2217 +漯 2218 +漱 2219 +漳 2220 +漾 2221 +潇 2222 +潘 2223 +潜 2224 +潞 2225 +潢 2226 +潭 2227 +潮 2228 +潼 2229 +澄 2230 +澈 2231 +澎 2232 +澜 2233 +澡 2234 +澳 2235 +激 2236 +濑 2237 +濒 2238 +濠 2239 +濡 2240 +濮 2241 +瀑 2242 +瀚 2243 +瀛 2244 +灌 2245 +灞 2246 +火 2247 +灭 2248 +灯 2249 +灰 2250 +灵 2251 +灶 2252 +灼 2253 +灾 2254 +灿 2255 +炅 2256 +炉 2257 +炊 2258 +炎 2259 +炒 2260 +炕 2261 +炖 2262 +炙 2263 +炜 2264 +炫 2265 +炬 2266 +炭 2267 +炮 2268 +炯 2269 +炳 2270 +炷 2271 +炸 2272 +点 2273 +炼 2274 +炽 2275 +烁 2276 +烂 2277 +烃 2278 +烈 2279 +烊 2280 +烘 2281 +烙 2282 +烟 2283 +烤 2284 +烦 2285 +烧 2286 +烨 2287 +烫 2288 +热 2289 +烯 2290 +烷 2291 +烹 2292 +烽 2293 +焉 2294 +焊 2295 +焕 2296 +焖 2297 +焘 2298 +焚 2299 +焦 2300 +焯 2301 +焰 2302 +焱 2303 +然 2304 +煊 2305 +煌 2306 +煎 2307 +煜 2308 +煞 2309 +煤 2310 +煦 2311 +照 2312 +煮 2313 +煲 2314 +熄 2315 +熊 2316 +熏 2317 +熔 2318 +熙 2319 +熟 2320 +熠 2321 +熨 2322 +熬 2323 +熹 2324 +燃 2325 +燊 2326 +燎 2327 +燕 2328 +燥 2329 +爆 2330 +爪 2331 +爬 2332 +爱 2333 +爵 2334 +父 2335 +爷 2336 +爸 2337 +爹 2338 +爽 2339 +片 2340 +版 2341 +牌 2342 +牙 2343 +牛 2344 +牟 2345 +牡 2346 +牢 2347 +牧 2348 +物 2349 +牲 2350 +牵 2351 +特 2352 +牺 2353 +牾 2354 +犀 2355 +犊 2356 +犒 2357 +犬 2358 +犯 2359 +状 2360 +犷 2361 +犹 2362 +狂 2363 +狄 2364 +狈 2365 +狐 2366 +狗 2367 +狙 2368 +狞 2369 +狠 2370 +狡 2371 +狩 2372 +独 2373 +狭 2374 +狮 2375 +狰 2376 +狱 2377 +狸 2378 +狼 2379 +猎 2380 +猖 2381 +猛 2382 +猜 2383 +猝 2384 +猥 2385 +猩 2386 +猪 2387 +猫 2388 +猬 2389 +献 2390 +猴 2391 +猾 2392 +猿 2393 +獒 2394 +獗 2395 +獾 2396 +玄 2397 +率 2398 +玉 2399 +王 2400 +玖 2401 +玛 2402 +玟 2403 +玥 2404 +玩 2405 +玫 2406 +玮 2407 +环 2408 +现 2409 +玲 2410 +玳 2411 +玺 2412 +玻 2413 +珀 2414 +珉 2415 +珊 2416 +珍 2417 +珏 2418 +珑 2419 +珜 2420 +珠 2421 +班 2422 +珮 2423 +珲 2424 +珺 2425 +球 2426 +琅 2427 +理 2428 +琉 2429 +琊 2430 +琏 2431 +琐 2432 +琛 2433 +琢 2434 +琥 2435 +琦 2436 +琪 2437 +琬 2438 +琰 2439 +琳 2440 +琴 2441 +琵 2442 +琶 2443 +琼 2444 +瑁 2445 +瑄 2446 +瑕 2447 +瑙 2448 +瑚 2449 +瑛 2450 +瑜 2451 +瑞 2452 +瑟 2453 +瑰 2454 +瑶 2455 +瑾 2456 +璀 2457 +璃 2458 +璇 2459 +璋 2460 +璐 2461 +璞 2462 +璧 2463 +璨 2464 +瓜 2465 +瓢 2466 +瓣 2467 +瓦 2468 +瓮 2469 +瓯 2470 +瓶 2471 +瓷 2472 +甄 2473 +甘 2474 +甚 2475 +甜 2476 +生 2477 +甥 2478 +用 2479 +甩 2480 +甫 2481 +甬 2482 +田 2483 +由 2484 +甲 2485 +申 2486 +电 2487 +男 2488 +甸 2489 +町 2490 +画 2491 +畅 2492 +畊 2493 +界 2494 +畏 2495 +畔 2496 +留 2497 +畜 2498 +略 2499 +番 2500 +畴 2501 +畸 2502 +畿 2503 +疃 2504 +疆 2505 +疏 2506 +疑 2507 +疗 2508 +疚 2509 +疝 2510 +疤 2511 +疫 2512 +疯 2513 +疲 2514 +疵 2515 +疹 2516 +疼 2517 +疾 2518 +病 2519 +症 2520 +痉 2521 +痊 2522 +痒 2523 +痕 2524 +痘 2525 +痛 2526 +痣 2527 +痪 2528 +痫 2529 +痰 2530 +痱 2531 +痴 2532 +痹 2533 +痼 2534 +瘀 2535 +瘁 2536 +瘟 2537 +瘠 2538 +瘤 2539 +瘦 2540 +瘩 2541 +瘪 2542 +瘫 2543 +瘸 2544 +瘾 2545 +癌 2546 +癖 2547 +癣 2548 +癫 2549 +登 2550 +白 2551 +百 2552 +皂 2553 +的 2554 +皆 2555 +皇 2556 +皋 2557 +皎 2558 +皓 2559 +皖 2560 +皙 2561 +皮 2562 +皱 2563 +盆 2564 +盈 2565 +益 2566 +盎 2567 +盐 2568 +监 2569 +盒 2570 +盔 2571 +盖 2572 +盗 2573 +盘 2574 +盛 2575 +盟 2576 +目 2577 +盯 2578 +盲 2579 +直 2580 +相 2581 +盹 2582 +盼 2583 +盾 2584 +省 2585 +眈 2586 +眉 2587 +看 2588 +真 2589 +眠 2590 +眨 2591 +眬 2592 +眯 2593 +眶 2594 +眷 2595 +眺 2596 +眼 2597 +着 2598 +睁 2599 +睐 2600 +睛 2601 +睡 2602 +督 2603 +睦 2604 +睫 2605 +睬 2606 +睹 2607 +睿 2608 +瞄 2609 +瞅 2610 +瞌 2611 +瞎 2612 +瞒 2613 +瞟 2614 +瞧 2615 +瞩 2616 +瞪 2617 +瞬 2618 +瞰 2619 +瞳 2620 +瞻 2621 +瞿 2622 +矗 2623 +矛 2624 +矜 2625 +矢 2626 +矣 2627 +知 2628 +矩 2629 +矫 2630 +短 2631 +矮 2632 +石 2633 +矶 2634 +矿 2635 +码 2636 +砂 2637 +砌 2638 +砍 2639 +砒 2640 +研 2641 +砖 2642 +砚 2643 +砝 2644 +砥 2645 +砰 2646 +砲 2647 +破 2648 +砷 2649 +砸 2650 +砺 2651 +砾 2652 +础 2653 +硅 2654 +硕 2655 +硚 2656 +硝 2657 +硫 2658 +硬 2659 +确 2660 +碉 2661 +碌 2662 +碍 2663 +碎 2664 +碑 2665 +碗 2666 +碘 2667 +碚 2668 +碟 2669 +碧 2670 +碰 2671 +碱 2672 +碳 2673 +碴 2674 +碾 2675 +磁 2676 +磅 2677 +磊 2678 +磋 2679 +磐 2680 +磕 2681 +磡 2682 +磨 2683 +磴 2684 +磷 2685 +磺 2686 +礁 2687 +示 2688 +礼 2689 +社 2690 +祁 2691 +祈 2692 +祉 2693 +祖 2694 +祛 2695 +祝 2696 +神 2697 +祠 2698 +祢 2699 +祥 2700 +票 2701 +祭 2702 +祯 2703 +祷 2704 +祸 2705 +祺 2706 +禀 2707 +禁 2708 +禄 2709 +禅 2710 +福 2711 +禧 2712 +禹 2713 +禺 2714 +离 2715 +禽 2716 +禾 2717 +秀 2718 +私 2719 +秃 2720 +秆 2721 +秉 2722 +秋 2723 +种 2724 +科 2725 +秒 2726 +秘 2727 +租 2728 +秣 2729 +秤 2730 +秦 2731 +秧 2732 +秩 2733 +积 2734 +称 2735 +秸 2736 +移 2737 +秽 2738 +稀 2739 +程 2740 +稍 2741 +税 2742 +稚 2743 +稠 2744 +稣 2745 +稳 2746 +稻 2747 +稼 2748 +稽 2749 +稿 2750 +穆 2751 +穗 2752 +穴 2753 +究 2754 +穷 2755 +空 2756 +穿 2757 +突 2758 +窃 2759 +窄 2760 +窈 2761 +窍 2762 +窑 2763 +窒 2764 +窕 2765 +窖 2766 +窗 2767 +窘 2768 +窜 2769 +窝 2770 +窟 2771 +窥 2772 +窦 2773 +窨 2774 +窿 2775 +立 2776 +竖 2777 +站 2778 +竞 2779 +竟 2780 +章 2781 +竣 2782 +童 2783 +竭 2784 +端 2785 +竲 2786 +竹 2787 +竺 2788 +竽 2789 +竿 2790 +笃 2791 +笈 2792 +笋 2793 +笑 2794 +笔 2795 +笙 2796 +笛 2797 +符 2798 +笨 2799 +第 2800 +笼 2801 +等 2802 +筋 2803 +筐 2804 +筑 2805 +筒 2806 +答 2807 +策 2808 +筛 2809 +筱 2810 +筵 2811 +筷 2812 +筹 2813 +签 2814 +简 2815 +箍 2816 +算 2817 +管 2818 +箫 2819 +箭 2820 +箱 2821 +篇 2822 +篡 2823 +篪 2824 +篮 2825 +篷 2826 +簇 2827 +簧 2828 +簸 2829 +簿 2830 +籁 2831 +籍 2832 +米 2833 +类 2834 +籽 2835 +粉 2836 +粒 2837 +粕 2838 +粗 2839 +粘 2840 +粟 2841 +粤 2842 +粥 2843 +粪 2844 +粮 2845 +粱 2846 +粹 2847 +精 2848 +糊 2849 +糕 2850 +糖 2851 +糗 2852 +糙 2853 +糟 2854 +糯 2855 +系 2856 +紊 2857 +素 2858 +索 2859 +紧 2860 +紫 2861 +累 2862 +絮 2863 +綦 2864 +繁 2865 +纠 2866 +红 2867 +纣 2868 +纤 2869 +约 2870 +级 2871 +纪 2872 +纬 2873 +纯 2874 +纰 2875 +纱 2876 +纲 2877 +纳 2878 +纵 2879 +纶 2880 +纷 2881 +纸 2882 +纹 2883 +纺 2884 +纽 2885 +线 2886 +练 2887 +组 2888 +绅 2889 +细 2890 +织 2891 +终 2892 +绊 2893 +绌 2894 +绍 2895 +绎 2896 +经 2897 +绑 2898 +绒 2899 +结 2900 +绕 2901 +绘 2902 +给 2903 +绚 2904 +络 2905 +绝 2906 +绞 2907 +统 2908 +绣 2909 +继 2910 +绩 2911 +绪 2912 +续 2913 +绮 2914 +绯 2915 +绰 2916 +绳 2917 +维 2918 +绵 2919 +绷 2920 +绸 2921 +综 2922 +绽 2923 +绿 2924 +缀 2925 +缄 2926 +缅 2927 +缆 2928 +缇 2929 +缉 2930 +缓 2931 +缔 2932 +缕 2933 +编 2934 +缘 2935 +缙 2936 +缚 2937 +缜 2938 +缝 2939 +缠 2940 +缤 2941 +缨 2942 +缩 2943 +缪 2944 +缭 2945 +缮 2946 +缰 2947 +缴 2948 +缸 2949 +缺 2950 +罂 2951 +罄 2952 +罐 2953 +网 2954 +罕 2955 +罗 2956 +罚 2957 +罡 2958 +罢 2959 +罩 2960 +罪 2961 +置 2962 +署 2963 +罹 2964 +羁 2965 +羊 2966 +美 2967 +羚 2968 +羞 2969 +羡 2970 +羣 2971 +群 2972 +羲 2973 +羹 2974 +羽 2975 +羿 2976 +翁 2977 +翅 2978 +翌 2979 +翔 2980 +翘 2981 +翟 2982 +翠 2983 +翡 2984 +翩 2985 +翰 2986 +翱 2987 +翻 2988 +翼 2989 +耀 2990 +老 2991 +考 2992 +耄 2993 +者 2994 +耋 2995 +而 2996 +耍 2997 +耐 2998 +耒 2999 +耕 3000 +耗 3001 +耘 3002 +耳 3003 +耶 3004 +耷 3005 +耸 3006 +耻 3007 +耽 3008 +耿 3009 +聂 3010 +聆 3011 +聊 3012 +聋 3013 +职 3014 +联 3015 +聘 3016 +聚 3017 +聪 3018 +肃 3019 +肆 3020 +肇 3021 +肉 3022 +肋 3023 +肌 3024 +肖 3025 +肘 3026 +肚 3027 +肛 3028 +肝 3029 +肠 3030 +股 3031 +肢 3032 +肤 3033 +肥 3034 +肩 3035 +肪 3036 +肮 3037 +肯 3038 +育 3039 +肴 3040 +肺 3041 +肾 3042 +肿 3043 +胀 3044 +胁 3045 +胃 3046 +胆 3047 +背 3048 +胎 3049 +胖 3050 +胚 3051 +胛 3052 +胜 3053 +胞 3054 +胡 3055 +胤 3056 +胧 3057 +胫 3058 +胯 3059 +胰 3060 +胱 3061 +胳 3062 +胶 3063 +胸 3064 +胺 3065 +能 3066 +脂 3067 +脆 3068 +脉 3069 +脊 3070 +脍 3071 +脏 3072 +脐 3073 +脑 3074 +脖 3075 +脚 3076 +脯 3077 +脱 3078 +脸 3079 +脾 3080 +腆 3081 +腊 3082 +腋 3083 +腌 3084 +腐 3085 +腑 3086 +腓 3087 +腔 3088 +腕 3089 +腥 3090 +腩 3091 +腰 3092 +腱 3093 +腹 3094 +腺 3095 +腻 3096 +腼 3097 +腾 3098 +腿 3099 +膀 3100 +膊 3101 +膏 3102 +膑 3103 +膛 3104 +膜 3105 +膝 3106 +膨 3107 +膳 3108 +膺 3109 +臀 3110 +臂 3111 +臃 3112 +臆 3113 +臣 3114 +自 3115 +臭 3116 +至 3117 +致 3118 +臻 3119 +舀 3120 +舅 3121 +舆 3122 +舌 3123 +舍 3124 +舒 3125 +舛 3126 +舜 3127 +舞 3128 +舟 3129 +航 3130 +般 3131 +舰 3132 +舱 3133 +舵 3134 +舶 3135 +舸 3136 +船 3137 +艇 3138 +艋 3139 +艘 3140 +良 3141 +艰 3142 +色 3143 +艳 3144 +艺 3145 +艾 3146 +节 3147 +芊 3148 +芋 3149 +芒 3150 +芙 3151 +芜 3152 +芝 3153 +芦 3154 +芬 3155 +芭 3156 +芮 3157 +芯 3158 +花 3159 +芳 3160 +芷 3161 +芸 3162 +芽 3163 +苇 3164 +苍 3165 +苏 3166 +苑 3167 +苗 3168 +苛 3169 +苟 3170 +苡 3171 +苣 3172 +若 3173 +苦 3174 +苯 3175 +英 3176 +苹 3177 +茁 3178 +茂 3179 +范 3180 +茄 3181 +茅 3182 +茆 3183 +茎 3184 +茗 3185 +茜 3186 +茨 3187 +茫 3188 +茵 3189 +茶 3190 +茸 3191 +茹 3192 +荃 3193 +荆 3194 +草 3195 +荐 3196 +荒 3197 +荔 3198 +荚 3199 +荞 3200 +荟 3201 +荡 3202 +荣 3203 +荤 3204 +荧 3205 +荫 3206 +药 3207 +荷 3208 +荼 3209 +莅 3210 +莆 3211 +莉 3212 +莎 3213 +莓 3214 +莘 3215 +莞 3216 +莠 3217 +莫 3218 +莱 3219 +莲 3220 +莴 3221 +获 3222 +莹 3223 +莺 3224 +莽 3225 +菁 3226 +菇 3227 +菊 3228 +菌 3229 +菜 3230 +菠 3231 +菡 3232 +菩 3233 +菱 3234 +菲 3235 +萃 3236 +萄 3237 +萋 3238 +萌 3239 +萍 3240 +萎 3241 +萝 3242 +萤 3243 +营 3244 +萦 3245 +萧 3246 +萨 3247 +萱 3248 +落 3249 +葆 3250 +著 3251 +葛 3252 +葡 3253 +董 3254 +葩 3255 +葫 3256 +葬 3257 +葱 3258 +葵 3259 +蒂 3260 +蒋 3261 +蒙 3262 +蒜 3263 +蒲 3264 +蒸 3265 +蒿 3266 +蓁 3267 +蓄 3268 +蓉 3269 +蓝 3270 +蓟 3271 +蓬 3272 +蔑 3273 +蔓 3274 +蔗 3275 +蔚 3276 +蔡 3277 +蔫 3278 +蔬 3279 +蔷 3280 +蔺 3281 +蔽 3282 +蕉 3283 +蕊 3284 +蕙 3285 +蕲 3286 +蕴 3287 +蕾 3288 +薄 3289 +薇 3290 +薛 3291 +薪 3292 +薯 3293 +薰 3294 +藏 3295 +藜 3296 +藤 3297 +藩 3298 +藻 3299 +蘑 3300 +虎 3301 +虐 3302 +虑 3303 +虚 3304 +虞 3305 +虫 3306 +虱 3307 +虹 3308 +虽 3309 +虾 3310 +蚀 3311 +蚁 3312 +蚂 3313 +蚊 3314 +蚌 3315 +蚓 3316 +蚕 3317 +蚝 3318 +蚣 3319 +蚯 3320 +蛀 3321 +蛇 3322 +蛋 3323 +蛐 3324 +蛙 3325 +蛛 3326 +蛟 3327 +蛮 3328 +蛰 3329 +蜀 3330 +蜂 3331 +蜇 3332 +蜈 3333 +蜊 3334 +蜒 3335 +蜓 3336 +蜕 3337 +蜘 3338 +蜚 3339 +蜜 3340 +蜡 3341 +蜥 3342 +蜴 3343 +蜷 3344 +蜿 3345 +蝇 3346 +蝉 3347 +蝎 3348 +蝗 3349 +蝙 3350 +蝠 3351 +蝴 3352 +蝶 3353 +螂 3354 +螃 3355 +融 3356 +螳 3357 +螺 3358 +蟑 3359 +蟹 3360 +蠢 3361 +血 3362 +衅 3363 +行 3364 +衍 3365 +衔 3366 +街 3367 +衙 3368 +衡 3369 +衣 3370 +补 3371 +表 3372 +衫 3373 +衬 3374 +衰 3375 +衷 3376 +袁 3377 +袂 3378 +袄 3379 +袆 3380 +袈 3381 +袋 3382 +袍 3383 +袒 3384 +袖 3385 +袜 3386 +被 3387 +袭 3388 +袱 3389 +裁 3390 +裂 3391 +装 3392 +裆 3393 +裔 3394 +裕 3395 +裙 3396 +裟 3397 +裤 3398 +裳 3399 +裴 3400 +裸 3401 +裹 3402 +褂 3403 +褒 3404 +褓 3405 +褚 3406 +褛 3407 +褪 3408 +褴 3409 +褶 3410 +襁 3411 +襄 3412 +襟 3413 +西 3414 +要 3415 +覃 3416 +覆 3417 +见 3418 +观 3419 +规 3420 +觅 3421 +视 3422 +览 3423 +觉 3424 +觊 3425 +觎 3426 +觐 3427 +觑 3428 +角 3429 +解 3430 +觥 3431 +触 3432 +言 3433 +詹 3434 +誉 3435 +誓 3436 +警 3437 +譬 3438 +计 3439 +订 3440 +认 3441 +讧 3442 +讨 3443 +让 3444 +讪 3445 +训 3446 +议 3447 +讯 3448 +记 3449 +讲 3450 +讳 3451 +讶 3452 +许 3453 +讹 3454 +论 3455 +讼 3456 +讽 3457 +设 3458 +访 3459 +诀 3460 +证 3461 +评 3462 +诅 3463 +识 3464 +诈 3465 +诉 3466 +诊 3467 +词 3468 +译 3469 +诓 3470 +试 3471 +诗 3472 +诙 3473 +诚 3474 +话 3475 +诞 3476 +诟 3477 +诠 3478 +诡 3479 +询 3480 +该 3481 +详 3482 +诧 3483 +诩 3484 +诫 3485 +诬 3486 +语 3487 +误 3488 +诱 3489 +诲 3490 +说 3491 +诵 3492 +诶 3493 +请 3494 +诸 3495 +诺 3496 +读 3497 +诽 3498 +课 3499 +诿 3500 +谀 3501 +谁 3502 +调 3503 +谅 3504 +谈 3505 +谊 3506 +谋 3507 +谌 3508 +谍 3509 +谎 3510 +谐 3511 +谑 3512 +谓 3513 +谕 3514 +谙 3515 +谚 3516 +谜 3517 +谢 3518 +谣 3519 +谤 3520 +谦 3521 +谨 3522 +谩 3523 +谬 3524 +谭 3525 +谱 3526 +谴 3527 +谷 3528 +豁 3529 +豆 3530 +豚 3531 +象 3532 +豪 3533 +豫 3534 +豹 3535 +貅 3536 +貉 3537 +貌 3538 +貔 3539 +贝 3540 +贞 3541 +负 3542 +贡 3543 +财 3544 +责 3545 +贤 3546 +败 3547 +账 3548 +货 3549 +质 3550 +贩 3551 +贪 3552 +贫 3553 +贬 3554 +购 3555 +贮 3556 +贯 3557 +贱 3558 +贴 3559 +贵 3560 +贷 3561 +贸 3562 +费 3563 +贺 3564 +贼 3565 +贾 3566 +贿 3567 +赁 3568 +赂 3569 +赃 3570 +资 3571 +赋 3572 +赌 3573 +赎 3574 +赏 3575 +赐 3576 +赔 3577 +赖 3578 +赘 3579 +赚 3580 +赛 3581 +赝 3582 +赞 3583 +赠 3584 +赡 3585 +赢 3586 +赣 3587 +赤 3588 +赦 3589 +赫 3590 +走 3591 +赴 3592 +赵 3593 +赶 3594 +起 3595 +趁 3596 +超 3597 +越 3598 +趋 3599 +趟 3600 +趣 3601 +足 3602 +趴 3603 +趸 3604 +趾 3605 +跃 3606 +跄 3607 +跆 3608 +跌 3609 +跑 3610 +跛 3611 +距 3612 +跟 3613 +跤 3614 +跨 3615 +跪 3616 +路 3617 +跳 3618 +践 3619 +跷 3620 +跺 3621 +跻 3622 +踉 3623 +踊 3624 +踏 3625 +踝 3626 +踞 3627 +踢 3628 +踩 3629 +踪 3630 +踵 3631 +踹 3632 +蹂 3633 +蹄 3634 +蹈 3635 +蹊 3636 +蹚 3637 +蹦 3638 +蹬 3639 +蹭 3640 +蹲 3641 +蹴 3642 +蹶 3643 +蹼 3644 +蹿 3645 +躁 3646 +躏 3647 +身 3648 +躬 3649 +躯 3650 +躲 3651 +躺 3652 +车 3653 +轧 3654 +轨 3655 +轩 3656 +转 3657 +轮 3658 +软 3659 +轰 3660 +轴 3661 +轶 3662 +轻 3663 +载 3664 +轿 3665 +较 3666 +辄 3667 +辅 3668 +辆 3669 +辈 3670 +辉 3671 +辍 3672 +辐 3673 +辑 3674 +输 3675 +辖 3676 +辗 3677 +辘 3678 +辙 3679 +辛 3680 +辜 3681 +辞 3682 +辟 3683 +辣 3684 +辨 3685 +辩 3686 +辫 3687 +辰 3688 +辱 3689 +边 3690 +辽 3691 +达 3692 +迁 3693 +迂 3694 +迄 3695 +迅 3696 +过 3697 +迈 3698 +迎 3699 +运 3700 +近 3701 +返 3702 +还 3703 +这 3704 +进 3705 +远 3706 +违 3707 +连 3708 +迟 3709 +迢 3710 +迥 3711 +迪 3712 +迫 3713 +迭 3714 +述 3715 +迷 3716 +迸 3717 +迹 3718 +追 3719 +退 3720 +送 3721 +适 3722 +逃 3723 +逅 3724 +逆 3725 +选 3726 +逊 3727 +逍 3728 +透 3729 +逐 3730 +递 3731 +途 3732 +逗 3733 +通 3734 +逛 3735 +逝 3736 +逞 3737 +速 3738 +造 3739 +逡 3740 +逢 3741 +逮 3742 +逵 3743 +逸 3744 +逻 3745 +逼 3746 +逾 3747 +遁 3748 +遂 3749 +遇 3750 +遍 3751 +遏 3752 +遐 3753 +道 3754 +遗 3755 +遛 3756 +遢 3757 +遣 3758 +遥 3759 +遨 3760 +遭 3761 +遮 3762 +遴 3763 +遵 3764 +避 3765 +邀 3766 +邂 3767 +邃 3768 +邋 3769 +邑 3770 +邓 3771 +邛 3772 +邝 3773 +邢 3774 +那 3775 +邦 3776 +邪 3777 +邬 3778 +邮 3779 +邯 3780 +邱 3781 +邵 3782 +邹 3783 +邺 3784 +邻 3785 +郁 3786 +郊 3787 +郎 3788 +郑 3789 +郜 3790 +郝 3791 +郡 3792 +部 3793 +郫 3794 +郭 3795 +郸 3796 +都 3797 +鄂 3798 +鄙 3799 +鄞 3800 +鄢 3801 +酋 3802 +酌 3803 +配 3804 +酒 3805 +酗 3806 +酝 3807 +酣 3808 +酪 3809 +酬 3810 +酯 3811 +酱 3812 +酵 3813 +酶 3814 +酷 3815 +酸 3816 +酿 3817 +醇 3818 +醉 3819 +醋 3820 +醍 3821 +醐 3822 +醒 3823 +醛 3824 +采 3825 +釉 3826 +释 3827 +里 3828 +重 3829 +野 3830 +量 3831 +金 3832 +釜 3833 +鉴 3834 +鏖 3835 +鑫 3836 +针 3837 +钉 3838 +钊 3839 +钓 3840 +钛 3841 +钝 3842 +钞 3843 +钟 3844 +钠 3845 +钢 3846 +钥 3847 +钦 3848 +钧 3849 +钩 3850 +钮 3851 +钰 3852 +钱 3853 +钵 3854 +钻 3855 +钾 3856 +铀 3857 +铁 3858 +铂 3859 +铃 3860 +铅 3861 +铆 3862 +铉 3863 +铎 3864 +铐 3865 +铜 3866 +铝 3867 +铠 3868 +铣 3869 +铨 3870 +铬 3871 +铭 3872 +铮 3873 +铰 3874 +铲 3875 +银 3876 +铸 3877 +铺 3878 +链 3879 +铿 3880 +销 3881 +锁 3882 +锂 3883 +锄 3884 +锅 3885 +锆 3886 +锈 3887 +锋 3888 +锌 3889 +锏 3890 +锐 3891 +错 3892 +锜 3893 +锟 3894 +锡 3895 +锢 3896 +锣 3897 +锤 3898 +锥 3899 +锦 3900 +锭 3901 +键 3902 +锯 3903 +锰 3904 +锵 3905 +锷 3906 +锹 3907 +锻 3908 +镀 3909 +镁 3910 +镇 3911 +镉 3912 +镊 3913 +镍 3914 +镑 3915 +镖 3916 +镜 3917 +镯 3918 +镳 3919 +镶 3920 +长 3921 +门 3922 +闪 3923 +闫 3924 +闭 3925 +问 3926 +闯 3927 +闰 3928 +闲 3929 +闳 3930 +间 3931 +闵 3932 +闷 3933 +闸 3934 +闹 3935 +闺 3936 +闻 3937 +闽 3938 +阀 3939 +阁 3940 +阂 3941 +阅 3942 +阎 3943 +阐 3944 +阔 3945 +阙 3946 +阚 3947 +阜 3948 +队 3949 +阮 3950 +阱 3951 +防 3952 +阳 3953 +阴 3954 +阵 3955 +阶 3956 +阻 3957 +阿 3958 +陀 3959 +陂 3960 +附 3961 +际 3962 +陆 3963 +陈 3964 +陋 3965 +陌 3966 +降 3967 +限 3968 +陕 3969 +陡 3970 +院 3971 +除 3972 +陨 3973 +险 3974 +陪 3975 +陬 3976 +陵 3977 +陶 3978 +陷 3979 +隅 3980 +隆 3981 +隋 3982 +隍 3983 +随 3984 +隐 3985 +隔 3986 +隘 3987 +隙 3988 +障 3989 +隧 3990 +隶 3991 +隼 3992 +隽 3993 +难 3994 +雀 3995 +雁 3996 +雄 3997 +雅 3998 +集 3999 +雇 4000 +雌 4001 +雍 4002 +雏 4003 +雕 4004 +雨 4005 +雪 4006 +雯 4007 +雳 4008 +零 4009 +雷 4010 +雾 4011 +需 4012 +霁 4013 +霄 4014 +霆 4015 +震 4016 +霈 4017 +霉 4018 +霍 4019 +霎 4020 +霏 4021 +霖 4022 +霜 4023 +霞 4024 +露 4025 +霸 4026 +霹 4027 +霾 4028 +靑 4029 +青 4030 +靓 4031 +靖 4032 +静 4033 +靛 4034 +非 4035 +靠 4036 +靡 4037 +面 4038 +革 4039 +靳 4040 +靴 4041 +靶 4042 +鞋 4043 +鞍 4044 +鞘 4045 +鞠 4046 +鞭 4047 +韦 4048 +韧 4049 +韩 4050 +韬 4051 +音 4052 +韵 4053 +韶 4054 +页 4055 +顶 4056 +顷 4057 +项 4058 +顺 4059 +须 4060 +顽 4061 +顾 4062 +顿 4063 +颁 4064 +颂 4065 +预 4066 +颅 4067 +领 4068 +颇 4069 +颈 4070 +颊 4071 +颍 4072 +颐 4073 +频 4074 +颓 4075 +颖 4076 +颗 4077 +题 4078 +颚 4079 +颜 4080 +额 4081 +颠 4082 +颤 4083 +风 4084 +飒 4085 +飓 4086 +飘 4087 +飙 4088 +飚 4089 +飞 4090 +食 4091 +餐 4092 +餮 4093 +饕 4094 +饥 4095 +饪 4096 +饭 4097 +饮 4098 +饰 4099 +饱 4100 +饲 4101 +饵 4102 +饶 4103 +饺 4104 +饼 4105 +饽 4106 +饿 4107 +馀 4108 +馅 4109 +馆 4110 +馈 4111 +馊 4112 +馋 4113 +馑 4114 +馒 4115 +首 4116 +馗 4117 +香 4118 +馥 4119 +馨 4120 +马 4121 +驭 4122 +驯 4123 +驰 4124 +驱 4125 +驳 4126 +驴 4127 +驶 4128 +驻 4129 +驼 4130 +驾 4131 +驿 4132 +骁 4133 +骂 4134 +骄 4135 +骅 4136 +骆 4137 +骇 4138 +骊 4139 +骋 4140 +验 4141 +骏 4142 +骐 4143 +骑 4144 +骗 4145 +骚 4146 +骜 4147 +骤 4148 +骥 4149 +骨 4150 +骷 4151 +骸 4152 +骼 4153 +髅 4154 +髋 4155 +髓 4156 +高 4157 +髦 4158 +鬼 4159 +魁 4160 +魂 4161 +魄 4162 +魅 4163 +魇 4164 +魏 4165 +魔 4166 +鱼 4167 +鲁 4168 +鲍 4169 +鲜 4170 +鲟 4171 +鲨 4172 +鲶 4173 +鲷 4174 +鲸 4175 +鳄 4176 +鳅 4177 +鳌 4178 +鳖 4179 +鳝 4180 +鳞 4181 +鸟 4182 +鸠 4183 +鸡 4184 +鸣 4185 +鸥 4186 +鸦 4187 +鸭 4188 +鸯 4189 +鸳 4190 +鸵 4191 +鸽 4192 +鸾 4193 +鸿 4194 +鹃 4195 +鹅 4196 +鹊 4197 +鹏 4198 +鹜 4199 +鹞 4200 +鹤 4201 +鹭 4202 +鹰 4203 +鹿 4204 +麋 4205 +麒 4206 +麓 4207 +麟 4208 +麦 4209 +麻 4210 +麾 4211 +黄 4212 +黍 4213 +黎 4214 +黏 4215 +黑 4216 +黔 4217 +默 4218 +黛 4219 +黝 4220 +黯 4221 +鼎 4222 +鼓 4223 +鼠 4224 +鼻 4225 +鼾 4226 +齐 4227 +齿 4228 +龄 4229 +龙 4230 +龚 4231 +龟 4232 diff --git a/examples/asr/aishell/local/prepare_data.py b/examples/asr/aishell/local/prepare_data.py new file mode 100644 index 00000000..53bb320c --- /dev/null +++ b/examples/asr/aishell/local/prepare_data.py @@ -0,0 +1,112 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" aishell dataset """ + +import os +import sys +import codecs +import pandas +from absl import logging + +import tensorflow as tf +from athena import get_wave_file_length + +SUBSETS = ["train", "dev", "test"] + + +def convert_audio_and_split_transcript(directory, subset, out_csv_file): + """Convert tar.gz to WAV and split the transcript. + + Args: + input_dir: the directory which holds the input dataset. + source_name: the name of the specified dataset. e.g. train/dev/test + output_dir: the directory to place the newly generated wav and json files. + out_json_file: the name of the newly generated json file. + out_vocab_file: the name of the generated vocab file. + """ + + gfile = tf.compat.v1.gfile + logging.info("Processing audio and transcript for {}".format(subset)) + audio_dir = os.path.join(directory, "wav/") + trans_dir = os.path.join(directory, "transcript/") + + files = [] + char_dict = {} + if not gfile.Exists(os.path.join(directory, subset)): # not unzip wav yet + for filename in os.listdir(audio_dir): + os.system("tar -zxvf " + audio_dir + filename + " -C " + directory) + + with codecs.open(os.path.join(trans_dir, "aishell_transcript_v0.8.txt"), "r", encoding="utf-8") as f: + for line in f: + items = line.strip().split(" ") + wav_filename = items[0] + labels = "" + for item in items[1:]: + labels += item + if item in char_dict: + char_dict[item] += 1 + else: + char_dict[item] = 0 + files.append((wav_filename + ".wav", labels)) + files_size_dict = {} + output_wav_dir = os.path.join(directory, subset) + + for root, subdirs, _ in gfile.Walk(output_wav_dir): + for subdir in subdirs: + for filename in os.listdir(os.path.join(root, subdir)): + files_size_dict[filename] = ( + get_wave_file_length(os.path.join(root, subdir, filename)), + subdir, + ) + + content = [] + for wav_filename, trans in files: + if wav_filename in files_size_dict: # wav which has trans is valid + filesize, subdir = files_size_dict[wav_filename] + abspath = os.path.join(output_wav_dir, subdir, wav_filename) + content.append((abspath, filesize, trans, subdir)) + files = content + + # Write to CSV file which contains three columns: + # "wav_filename", "wav_length_ms", "transcript", "speakers". + df = pandas.DataFrame( + data=files, columns=["wav_filename", "wav_length_ms", "transcript", "speaker"] + ) + df.to_csv(out_csv_file, index=False, sep="\t") + logging.info("Successfully generated csv file {}".format(out_csv_file)) + +def processor(dircetory, subset, force_process): + """ download and process """ + if subset not in SUBSETS: + raise ValueError(subset, "is not in AISHELL") + if force_process: + logging.info("force process is set to be true") + + subset_csv = os.path.join(dircetory, subset + ".csv") + if not force_process and os.path.exists(subset_csv): + logging.info("{} already exist".format(subset_csv)) + return subset_csv + logging.info("Processing the AISHELL subset {} in {}".format(subset, dircetory)) + convert_audio_and_split_transcript(dircetory, subset, subset_csv) + logging.info("Finished processing AISHELL subset {}".format(subset)) + return subset_csv + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + DIR = sys.argv[1] + for SUBSET in SUBSETS: + processor(DIR, SUBSET, True) + diff --git a/examples/asr/aishell/mtl_transformer.json b/examples/asr/aishell/mtl_transformer.json new file mode 100644 index 00000000..2f0d58ba --- /dev/null +++ b/examples/asr/aishell/mtl_transformer.json @@ -0,0 +1,70 @@ +{ + "batch_size":32, + "num_epochs":15, + "sorta_epoch":1, + "ckpt":"examples/asr/aishell/ckpts/mtl_transformer_ctc/", + "summary_dir":"examples/asr/aishell/ckpts/mtl_transformer_ctc/event", + + "solver_gpu":[2], + "solver_config":{ + "clip_norm":100, + "log_interval":10, + "enable_tf_function":true + }, + + "model":"mtl_transformer_ctc", + "num_classes": null, + "pretrained_model": null, + "model_config":{ + "model":"speech_transformer", + "model_config":{ + "return_encoder_output":true, + "num_filters":512, + "d_model":512, + "num_heads":8, + "num_encoder_layers":12, + "num_decoder_layers":6, + "dff":1280, + "rate":0.1, + "label_smoothing_rate":0.0, + "schedual_sampling_rate":0.9 + }, + "mtl_weight":0.5 + }, + + "decode_config":{ + "beam_search":true, + "beam_size":4, + "ctc_weight":0.3, + "lm_weight":0.1, + "lm_path":"examples/asr/aishell/data/lm.bin" + }, + + "optimizer":"warmup_adam", + "optimizer_config":{ + "d_model":512, + "warmup_steps":8000, + "k":0.5, + "decay_steps": 50000, + "decay_rate": 0.1 + }, + + "dataset_builder": "speech_recognition_dataset", + "dataset_config":{ + "audio_config":{ + "type":"Fbank", + "filterbank_channel_count":40, + "local_cmvn":false + }, + "cmvn_file":"examples/asr/aishell/data/cmvn", + "text_config": { + "type":"vocab", + "model":"examples/asr/aishell/data/vocab" + }, + "input_length_range":[10, 8000] + }, + "num_data_threads": 1, + "train_csv":"examples/asr/aishell/data/train.csv", + "dev_csv":"examples/asr/aishell/data/dev.csv", + "test_csv":"examples/asr/aishell/data/test.csv" +} diff --git a/examples/asr/aishell/mtl_transformer_sp.json b/examples/asr/aishell/mtl_transformer_sp.json new file mode 100644 index 00000000..25ea9c52 --- /dev/null +++ b/examples/asr/aishell/mtl_transformer_sp.json @@ -0,0 +1,71 @@ +{ + "batch_size":32, + "num_epochs":15, + "sorta_epoch":1, + "ckpt":"examples/asr/aishell/ckpts/mtl_transformer_ctc/", + "summary_dir":"examples/asr/aishell/ckpts/mtl_transformer_ctc/event", + + "solver_gpu":[2], + "solver_config":{ + "clip_norm":100, + "log_interval":10, + "enable_tf_function":true + }, + + "model":"mtl_transformer_ctc", + "num_classes": null, + "pretrained_model": null, + "model_config":{ + "model":"speech_transformer", + "model_config":{ + "return_encoder_output":true, + "num_filters":512, + "d_model":512, + "num_heads":8, + "num_encoder_layers":12, + "num_decoder_layers":6, + "dff":1280, + "rate":0.1, + "label_smoothing_rate":0.0, + "schedual_sampling_rate":0.9 + }, + "mtl_weight":0.5 + }, + + "decode_config":{ + "beam_search":true, + "beam_size":4, + "ctc_weight":0.3, + "lm_weight":0.1, + "lm_path":"examples/asr/aishell/data/lm.bin" + }, + + "optimizer":"warmup_adam", + "optimizer_config":{ + "d_model":512, + "warmup_steps":3500, + "k":0.7, + "decay_steps": 100000, + "decay_rate": 0.1 + }, + + "dataset_builder": "speech_recognition_dataset", + "dataset_config":{ + "audio_config":{ + "type":"Fbank", + "filterbank_channel_count":40, + "local_cmvn":false + }, + "cmvn_file":"examples/asr/aishell/data/cmvn", + "text_config": { + "type":"vocab", + "model":"examples/asr/aishell/data/vocab" + }, + "input_length_range":[10, 8000], + "speed_permutation": [0.9, 1.0, 1.1] + }, + "num_data_threads": 1, + "train_csv":"examples/asr/aishell/data/train.csv", + "dev_csv":"examples/asr/aishell/data/dev.csv", + "test_csv":"examples/asr/aishell/data/test.csv" +} diff --git a/examples/asr/hkust/README.md b/examples/asr/hkust/README.md new file mode 100644 index 00000000..3a4867f1 --- /dev/null +++ b/examples/asr/hkust/README.md @@ -0,0 +1,18 @@ + +# Examples for HKUST + +## 1 Transformer + +```bash +source env.sh +python examples/asr/hkust/prepare_data.py /tmp-data/dataset/opensource/hkust +python athena/main.py examples/asr/hkust/transformer.json +``` + +## 2 MTL_Transformer_CTC + +```bash +source env.sh +python examples/asr/hkust/prepare_data.py /tmp-data/dataset/opensource/hkust +python athena/main.py examples/asr/hkust/mtl_transformer.json +``` diff --git a/examples/asr/hkust/data/vocab b/examples/asr/hkust/data/vocab new file mode 100644 index 00000000..50d55ffd --- /dev/null +++ b/examples/asr/hkust/data/vocab @@ -0,0 +1,3650 @@ + 0 +一 1 +丁 2 +七 3 +万 4 +丈 5 +三 6 +上 7 +下 8 +不 9 +与 10 +丐 11 +丑 12 +专 13 +且 14 +世 15 +丘 16 +业 17 +丛 18 +东 19 +丝 20 +丞 21 +丢 22 +两 23 +严 24 +丧 25 +个 26 +丫 27 +中 28 +丰 29 +串 30 +临 31 +丸 32 +丹 33 +为 34 +主 35 +丽 36 +举 37 +乃 38 +久 39 +么 40 +义 41 +之 42 +乌 43 +乎 44 +乏 45 +乐 46 +乒 47 +乓 48 +乔 49 +乖 50 +乘 51 +乙 52 +九 53 +乞 54 +也 55 +习 56 +乡 57 +书 58 +买 59 +乱 60 +乳 61 +乾 62 +了 63 +予 64 +争 65 +事 66 +二 67 +于 68 +亏 69 +云 70 +互 71 +五 72 +井 73 +亚 74 +些 75 +亡 76 +亢 77 +交 78 +亦 79 +产 80 +亨 81 +亩 82 +享 83 +京 84 +亭 85 +亮 86 +亲 87 +亳 88 +亵 89 +人 90 +亿 91 +什 92 +仁 93 +仃 94 +仄 95 +仅 96 +仇 97 +今 98 +介 99 +仍 100 +从 101 +仑 102 +仓 103 +仔 104 +他 105 +仗 106 +付 107 +仙 108 +代 109 +令 110 +以 111 +仨 112 +仪 113 +们 114 +仰 115 +仲 116 +件 117 +价 118 +任 119 +份 120 +仿 121 +企 122 +伊 123 +伍 124 +伏 125 +伐 126 +休 127 +众 128 +优 129 +伙 130 +会 131 +伞 132 +伟 133 +传 134 +伤 135 +伦 136 +伪 137 +伯 138 +估 139 +伴 140 +伶 141 +伸 142 +伺 143 +似 144 +伽 145 +佃 146 +但 147 +位 148 +低 149 +住 150 +佐 151 +佑 152 +体 153 +何 154 +佗 155 +余 156 +佛 157 +作 158 +你 159 +佩 160 +佬 161 +佰 162 +佳 163 +使 164 +侃 165 +侄 166 +侈 167 +例 168 +供 169 +依 170 +侠 171 +侣 172 +侥 173 +侦 174 +侧 175 +侨 176 +侮 177 +侯 178 +侵 179 +便 180 +促 181 +俄 182 +俊 183 +俏 184 +俐 185 +俑 186 +俗 187 +俘 188 +保 189 +俞 190 +俟 191 +信 192 +俩 193 +俭 194 +修 195 +俯 196 +俱 197 +俺 198 +倍 199 +倒 200 +倔 201 +倘 202 +候 203 +倚 204 +倜 205 +借 206 +倡 207 +倦 208 +倩 209 +倪 210 +倭 211 +债 212 +值 213 +倾 214 +假 215 +偏 216 +做 217 +停 218 +健 219 +偶 220 +偷 221 +偾 222 +偿 223 +傀 224 +傅 225 +傍 226 +傣 227 +傥 228 +储 229 +催 230 +傲 231 +傻 232 +像 233 +僚 234 +僧 235 +僵 236 +僻 237 +儒 238 +儡 239 +儿 240 +允 241 +元 242 +兄 243 +充 244 +兆 245 +先 246 +光 247 +克 248 +免 249 +兔 250 +党 251 +兜 252 +入 253 +全 254 +八 255 +公 256 +六 257 +兮 258 +兰 259 +共 260 +关 261 +兴 262 +兵 263 +其 264 +具 265 +典 266 +养 267 +兼 268 +兽 269 +内 270 +冉 271 +册 272 +再 273 +冒 274 +冕 275 +写 276 +军 277 +农 278 +冠 279 +冤 280 +冥 281 +冬 282 +冯 283 +冰 284 +冲 285 +决 286 +况 287 +冶 288 +冷 289 +冻 290 +净 291 +凄 292 +准 293 +凉 294 +凋 295 +凌 296 +减 297 +凑 298 +凝 299 +几 300 +凡 301 +凤 302 +凭 303 +凯 304 +凰 305 +凳 306 +凶 307 +凸 308 +出 309 +击 310 +函 311 +凿 312 +刀 313 +刁 314 +分 315 +切 316 +刊 317 +刑 318 +划 319 +列 320 +刘 321 +则 322 +刚 323 +创 324 +初 325 +删 326 +判 327 +刨 328 +利 329 +别 330 +刮 331 +到 332 +制 333 +刷 334 +券 335 +刹 336 +刺 337 +刻 338 +剂 339 +剃 340 +削 341 +前 342 +剑 343 +剔 344 +剖 345 +剥 346 +剧 347 +剩 348 +剪 349 +副 350 +割 351 +剿 352 +劈 353 +力 354 +劝 355 +办 356 +功 357 +加 358 +务 359 +劣 360 +动 361 +助 362 +努 363 +劫 364 +励 365 +劲 366 +劳 367 +势 368 +勃 369 +勇 370 +勉 371 +勋 372 +勒 373 +勤 374 +勺 375 +勾 376 +勿 377 +匀 378 +包 379 +匆 380 +匈 381 +化 382 +北 383 +匙 384 +匝 385 +匠 386 +匡 387 +匪 388 +匹 389 +区 390 +医 391 +匿 392 +十 393 +千 394 +升 395 +午 396 +卉 397 +半 398 +华 399 +协 400 +卑 401 +卒 402 +卓 403 +单 404 +卖 405 +南 406 +博 407 +卜 408 +占 409 +卡 410 +卢 411 +卤 412 +卦 413 +卧 414 +卫 415 +印 416 +危 417 +即 418 +却 419 +卵 420 +卷 421 +卸 422 +厂 423 +厅 424 +历 425 +厉 426 +压 427 +厌 428 +厕 429 +厘 430 +厚 431 +原 432 +厢 433 +厥 434 +厦 435 +厨 436 +去 437 +县 438 +参 439 +又 440 +叉 441 +及 442 +友 443 +双 444 +反 445 +发 446 +叔 447 +取 448 +受 449 +变 450 +叙 451 +叛 452 +叠 453 +口 454 +古 455 +句 456 +另 457 +叨 458 +只 459 +叫 460 +召 461 +叭 462 +叮 463 +可 464 +台 465 +史 466 +右 467 +叶 468 +号 469 +司 470 +叹 471 +叻 472 +叼 473 +叽 474 +吁 475 +吃 476 +各 477 +合 478 +吉 479 +吊 480 +同 481 +名 482 +后 483 +吏 484 +吐 485 +向 486 +吓 487 +吕 488 +吗 489 +君 490 +吝 491 +吞 492 +吟 493 +否 494 +吧 495 +吨 496 +吩 497 +含 498 +听 499 +吭 500 +启 501 +吱 502 +吴 503 +吵 504 +吸 505 +吹 506 +吻 507 +吼 508 +吾 509 +呀 510 +呃 511 +呆 512 +呈 513 +告 514 +呐 515 +呓 516 +呕 517 +呖 518 +呗 519 +员 520 +呛 521 +呜 522 +呢 523 +呦 524 +周 525 +呱 526 +味 527 +呵 528 +呸 529 +呼 530 +命 531 +咋 532 +和 533 +咎 534 +咏 535 +咐 536 +咒 537 +咔 538 +咕 539 +咖 540 +咙 541 +咚 542 +咣 543 +咦 544 +咧 545 +咨 546 +咪 547 +咬 548 +咯 549 +咱 550 +咳 551 +咸 552 +咽 553 +咿 554 +哀 555 +品 556 +哄 557 +哆 558 +哇 559 +哈 560 +哉 561 +响 562 +哎 563 +哐 564 +哑 565 +哒 566 +哓 567 +哗 568 +哟 569 +哥 570 +哦 571 +哧 572 +哨 573 +哩 574 +哪 575 +哭 576 +哮 577 +哲 578 +哼 579 +哽 580 +唇 581 +唉 582 +唏 583 +唐 584 +唔 585 +唠 586 +唤 587 +唧 588 +唬 589 +售 590 +唯 591 +唰 592 +唱 593 +唷 594 +唾 595 +啃 596 +商 597 +啊 598 +啡 599 +啤 600 +啥 601 +啦 602 +啪 603 +啬 604 +啰 605 +啵 606 +啷 607 +喀 608 +喂 609 +善 610 +喆 611 +喇 612 +喉 613 +喊 614 +喏 615 +喔 616 +喘 617 +喜 618 +喝 619 +喧 620 +喱 621 +喳 622 +喷 623 +喻 624 +喽 625 +嗑 626 +嗓 627 +嗜 628 +嗝 629 +嗡 630 +嗦 631 +嗨 632 +嗯 633 +嗲 634 +嗷 635 +嗽 636 +嘀 637 +嘈 638 +嘉 639 +嘎 640 +嘘 641 +嘛 642 +嘞 643 +嘟 644 +嘠 645 +嘣 646 +嘭 647 +嘱 648 +嘲 649 +嘴 650 +嘶 651 +嘻 652 +嘿 653 +噌 654 +噎 655 +噔 656 +噜 657 +噢 658 +器 659 +噪 660 +噻 661 +噼 662 +嚓 663 +嚣 664 +嚤 665 +嚯 666 +嚷 667 +嚼 668 +囊 669 +囚 670 +四 671 +回 672 +因 673 +团 674 +囫 675 +园 676 +困 677 +围 678 +囵 679 +固 680 +国 681 +图 682 +圆 683 +圈 684 +土 685 +圣 686 +在 687 +地 688 +圳 689 +场 690 +圾 691 +址 692 +均 693 +坊 694 +坎 695 +坏 696 +坐 697 +坑 698 +块 699 +坚 700 +坛 701 +坝 702 +坞 703 +坟 704 +坠 705 +坡 706 +坤 707 +坦 708 +坪 709 +坷 710 +垂 711 +垃 712 +垄 713 +型 714 +垒 715 +垢 716 +垫 717 +垮 718 +埃 719 +埋 720 +城 721 +域 722 +埠 723 +培 724 +基 725 +堂 726 +堆 727 +堑 728 +堕 729 +堡 730 +堤 731 +堪 732 +堵 733 +塌 734 +塑 735 +塔 736 +塘 737 +塞 738 +填 739 +境 740 +墅 741 +墉 742 +墓 743 +墙 744 +增 745 +墨 746 +墩 747 +壁 748 +壕 749 +壤 750 +士 751 +壮 752 +声 753 +壳 754 +壶 755 +处 756 +备 757 +复 758 +夏 759 +夕 760 +外 761 +多 762 +夜 763 +够 764 +大 765 +天 766 +太 767 +夫 768 +央 769 +失 770 +头 771 +夷 772 +夸 773 +夹 774 +夺 775 +奇 776 +奈 777 +奉 778 +奋 779 +奏 780 +契 781 +奔 782 +奕 783 +奖 784 +套 785 +奘 786 +奠 787 +奢 788 +奥 789 +女 790 +奴 791 +奶 792 +奷 793 +奸 794 +她 795 +好 796 +如 797 +妃 798 +妄 799 +妆 800 +妇 801 +妈 802 +妍 803 +妒 804 +妓 805 +妖 806 +妙 807 +妞 808 +妥 809 +妨 810 +妮 811 +妲 812 +妹 813 +妻 814 +妾 815 +姆 816 +姊 817 +始 818 +姐 819 +姑 820 +姓 821 +委 822 +姚 823 +姜 824 +姝 825 +姣 826 +姥 827 +姨 828 +姬 829 +姻 830 +姿 831 +威 832 +娃 833 +娇 834 +娘 835 +娜 836 +娟 837 +娥 838 +娱 839 +娴 840 +娶 841 +娼 842 +婆 843 +婉 844 +婊 845 +婚 846 +婪 847 +婴 848 +婷 849 +婿 850 +媒 851 +媚 852 +媛 853 +媲 854 +媳 855 +嫁 856 +嫂 857 +嫉 858 +嫌 859 +嫖 860 +嫣 861 +嫦 862 +嫩 863 +嬅 864 +嬉 865 +子 866 +孔 867 +孕 868 +字 869 +存 870 +孙 871 +孜 872 +孝 873 +孟 874 +季 875 +孤 876 +学 877 +孩 878 +孰 879 +孽 880 +宁 881 +它 882 +宅 883 +宇 884 +守 885 +安 886 +宋 887 +完 888 +宏 889 +宗 890 +官 891 +宙 892 +定 893 +宛 894 +宜 895 +宝 896 +实 897 +宠 898 +审 899 +客 900 +宣 901 +室 902 +宦 903 +宪 904 +宫 905 +宰 906 +害 907 +宴 908 +宵 909 +家 910 +容 911 +宽 912 +宾 913 +宿 914 +寂 915 +寄 916 +密 917 +寇 918 +富 919 +寒 920 +寓 921 +寝 922 +寞 923 +察 924 +寡 925 +寥 926 +寨 927 +寸 928 +对 929 +寺 930 +寻 931 +导 932 +寿 933 +封 934 +射 935 +将 936 +尊 937 +小 938 +少 939 +尔 940 +尖 941 +尘 942 +尚 943 +尝 944 +尢 945 +尤 946 +尬 947 +就 948 +尴 949 +尸 950 +尹 951 +尺 952 +尼 953 +尽 954 +尾 955 +尿 956 +局 957 +屁 958 +层 959 +居 960 +屈 961 +届 962 +屋 963 +屌 964 +屎 965 +屏 966 +展 967 +属 968 +屠 969 +屡 970 +履 971 +屯 972 +山 973 +屹 974 +屿 975 +岁 976 +岂 977 +岔 978 +岗 979 +岚 980 +岛 981 +岩 982 +岭 983 +岳 984 +岸 985 +峙 986 +峡 987 +峨 988 +峪 989 +峭 990 +峰 991 +峻 992 +崂 993 +崇 994 +崎 995 +崔 996 +崖 997 +崛 998 +崩 999 +嵋 1000 +嵩 1001 +嵬 1002 +巍 1003 +川 1004 +州 1005 +巡 1006 +巢 1007 +工 1008 +左 1009 +巧 1010 +巨 1011 +巩 1012 +巫 1013 +差 1014 +己 1015 +已 1016 +巴 1017 +巷 1018 +巾 1019 +币 1020 +市 1021 +布 1022 +帅 1023 +帆 1024 +师 1025 +希 1026 +帐 1027 +帕 1028 +帖 1029 +帘 1030 +帜 1031 +帝 1032 +带 1033 +席 1034 +帮 1035 +常 1036 +帼 1037 +帽 1038 +幅 1039 +幌 1040 +幕 1041 +幢 1042 +干 1043 +平 1044 +年 1045 +并 1046 +幸 1047 +幺 1048 +幻 1049 +幼 1050 +幽 1051 +广 1052 +庄 1053 +庆 1054 +庇 1055 +床 1056 +序 1057 +庐 1058 +库 1059 +应 1060 +底 1061 +店 1062 +庙 1063 +庚 1064 +府 1065 +庞 1066 +废 1067 +度 1068 +座 1069 +庭 1070 +庵 1071 +庶 1072 +康 1073 +庸 1074 +廉 1075 +廊 1076 +廓 1077 +廖 1078 +延 1079 +廷 1080 +建 1081 +开 1082 +异 1083 +弃 1084 +弄 1085 +弊 1086 +式 1087 +弓 1088 +引 1089 +弗 1090 +弘 1091 +弟 1092 +张 1093 +弥 1094 +弦 1095 +弧 1096 +弩 1097 +弯 1098 +弱 1099 +弹 1100 +强 1101 +归 1102 +当 1103 +录 1104 +彗 1105 +彝 1106 +形 1107 +彤 1108 +彦 1109 +彩 1110 +彪 1111 +彬 1112 +彭 1113 +影 1114 +役 1115 +彻 1116 +彼 1117 +往 1118 +征 1119 +径 1120 +待 1121 +很 1122 +徊 1123 +律 1124 +徐 1125 +徒 1126 +得 1127 +徘 1128 +徙 1129 +御 1130 +循 1131 +微 1132 +德 1133 +徽 1134 +心 1135 +必 1136 +忆 1137 +忌 1138 +忍 1139 +忏 1140 +忒 1141 +志 1142 +忘 1143 +忙 1144 +忠 1145 +忧 1146 +快 1147 +念 1148 +忽 1149 +忿 1150 +怀 1151 +态 1152 +怅 1153 +怆 1154 +怎 1155 +怒 1156 +怕 1157 +怖 1158 +怜 1159 +思 1160 +怠 1161 +怡 1162 +急 1163 +性 1164 +怨 1165 +怪 1166 +总 1167 +恋 1168 +恍 1169 +恐 1170 +恒 1171 +恕 1172 +恢 1173 +恤 1174 +恨 1175 +恩 1176 +恭 1177 +息 1178 +恰 1179 +恳 1180 +恶 1181 +恺 1182 +恼 1183 +恿 1184 +悄 1185 +悉 1186 +悍 1187 +悔 1188 +悚 1189 +悟 1190 +悠 1191 +患 1192 +悦 1193 +您 1194 +悬 1195 +悲 1196 +悴 1197 +情 1198 +惊 1199 +惋 1200 +惑 1201 +惕 1202 +惜 1203 +惠 1204 +惦 1205 +惧 1206 +惨 1207 +惩 1208 +惫 1209 +惬 1210 +惭 1211 +惮 1212 +惯 1213 +惰 1214 +想 1215 +惹 1216 +愁 1217 +愉 1218 +意 1219 +愚 1220 +感 1221 +愣 1222 +愤 1223 +愧 1224 +愿 1225 +慈 1226 +慌 1227 +慎 1228 +慑 1229 +慕 1230 +慢 1231 +慧 1232 +慨 1233 +慰 1234 +慷 1235 +憋 1236 +憎 1237 +憔 1238 +憧 1239 +憬 1240 +憾 1241 +懂 1242 +懈 1243 +懒 1244 +懦 1245 +懵 1246 +戈 1247 +戏 1248 +成 1249 +我 1250 +戒 1251 +或 1252 +战 1253 +戚 1254 +截 1255 +戳 1256 +戴 1257 +户 1258 +房 1259 +所 1260 +扁 1261 +扇 1262 +扈 1263 +扉 1264 +手 1265 +才 1266 +扎 1267 +扑 1268 +扒 1269 +打 1270 +扔 1271 +托 1272 +扛 1273 +扣 1274 +执 1275 +扩 1276 +扫 1277 +扬 1278 +扭 1279 +扮 1280 +扯 1281 +扰 1282 +扳 1283 +扶 1284 +批 1285 +扼 1286 +找 1287 +承 1288 +技 1289 +抄 1290 +抉 1291 +把 1292 +抑 1293 +抒 1294 +抓 1295 +投 1296 +抖 1297 +抗 1298 +折 1299 +抚 1300 +抛 1301 +抠 1302 +抢 1303 +护 1304 +报 1305 +披 1306 +抬 1307 +抱 1308 +抵 1309 +抹 1310 +押 1311 +抽 1312 +担 1313 +拆 1314 +拉 1315 +拌 1316 +拍 1317 +拎 1318 +拐 1319 +拒 1320 +拓 1321 +拔 1322 +拖 1323 +拘 1324 +拙 1325 +招 1326 +拜 1327 +拟 1328 +拢 1329 +拣 1330 +拥 1331 +拦 1332 +拧 1333 +拨 1334 +择 1335 +括 1336 +拭 1337 +拮 1338 +拯 1339 +拳 1340 +拴 1341 +拷 1342 +拼 1343 +拽 1344 +拾 1345 +拿 1346 +持 1347 +挂 1348 +指 1349 +按 1350 +挎 1351 +挑 1352 +挖 1353 +挚 1354 +挟 1355 +挠 1356 +挡 1357 +挣 1358 +挤 1359 +挥 1360 +挨 1361 +挪 1362 +挫 1363 +振 1364 +挺 1365 +挽 1366 +捂 1367 +捅 1368 +捆 1369 +捉 1370 +捍 1371 +捎 1372 +捏 1373 +捐 1374 +捕 1375 +捞 1376 +损 1377 +捡 1378 +换 1379 +捣 1380 +捧 1381 +据 1382 +捱 1383 +捶 1384 +捷 1385 +掀 1386 +授 1387 +掉 1388 +掌 1389 +掏 1390 +掐 1391 +排 1392 +掘 1393 +掠 1394 +探 1395 +接 1396 +控 1397 +推 1398 +掩 1399 +措 1400 +掰 1401 +掷 1402 +掺 1403 +揉 1404 +揍 1405 +描 1406 +提 1407 +插 1408 +握 1409 +揣 1410 +揪 1411 +揭 1412 +援 1413 +揽 1414 +搁 1415 +搂 1416 +搅 1417 +搏 1418 +搓 1419 +搛 1420 +搜 1421 +搞 1422 +搪 1423 +搬 1424 +搭 1425 +携 1426 +搽 1427 +摁 1428 +摄 1429 +摆 1430 +摇 1431 +摈 1432 +摊 1433 +摔 1434 +摘 1435 +摞 1436 +摧 1437 +摩 1438 +摸 1439 +撂 1440 +撇 1441 +撑 1442 +撒 1443 +撕 1444 +撞 1445 +撤 1446 +撩 1447 +撬 1448 +播 1449 +撮 1450 +撰 1451 +撵 1452 +撼 1453 +擀 1454 +擅 1455 +操 1456 +擦 1457 +攀 1458 +攒 1459 +攥 1460 +支 1461 +收 1462 +改 1463 +攻 1464 +放 1465 +政 1466 +故 1467 +效 1468 +敌 1469 +敏 1470 +救 1471 +教 1472 +敛 1473 +敞 1474 +敢 1475 +散 1476 +敦 1477 +敬 1478 +数 1479 +敲 1480 +整 1481 +敷 1482 +文 1483 +斋 1484 +斌 1485 +斐 1486 +斑 1487 +斓 1488 +斗 1489 +料 1490 +斜 1491 +斤 1492 +斥 1493 +斧 1494 +斩 1495 +断 1496 +斯 1497 +新 1498 +方 1499 +施 1500 +旁 1501 +旅 1502 +旋 1503 +旎 1504 +族 1505 +旖 1506 +旗 1507 +无 1508 +既 1509 +日 1510 +旦 1511 +旧 1512 +旨 1513 +早 1514 +旬 1515 +旭 1516 +旮 1517 +旯 1518 +旱 1519 +时 1520 +旷 1521 +旺 1522 +昂 1523 +昆 1524 +昊 1525 +昌 1526 +明 1527 +昏 1528 +易 1529 +昔 1530 +昕 1531 +昙 1532 +星 1533 +映 1534 +春 1535 +昧 1536 +昨 1537 +昭 1538 +是 1539 +昵 1540 +昼 1541 +显 1542 +晃 1543 +晋 1544 +晏 1545 +晒 1546 +晓 1547 +晕 1548 +晚 1549 +晦 1550 +晨 1551 +普 1552 +景 1553 +晰 1554 +晴 1555 +晶 1556 +智 1557 +晾 1558 +暂 1559 +暇 1560 +暑 1561 +暖 1562 +暗 1563 +暧 1564 +暴 1565 +曲 1566 +更 1567 +曹 1568 +曼 1569 +曾 1570 +替 1571 +最 1572 +月 1573 +有 1574 +朋 1575 +服 1576 +朔 1577 +朗 1578 +望 1579 +朝 1580 +期 1581 +朦 1582 +木 1583 +未 1584 +末 1585 +本 1586 +术 1587 +朱 1588 +朴 1589 +朵 1590 +机 1591 +杀 1592 +杂 1593 +权 1594 +杆 1595 +杉 1596 +李 1597 +杏 1598 +材 1599 +村 1600 +杖 1601 +杜 1602 +杞 1603 +束 1604 +杠 1605 +条 1606 +来 1607 +杨 1608 +杭 1609 +杯 1610 +杰 1611 +杳 1612 +杵 1613 +松 1614 +板 1615 +极 1616 +构 1617 +枉 1618 +析 1619 +枕 1620 +林 1621 +枚 1622 +果 1623 +枝 1624 +枢 1625 +枣 1626 +枪 1627 +枫 1628 +枭 1629 +枯 1630 +架 1631 +枷 1632 +柄 1633 +柏 1634 +某 1635 +染 1636 +柔 1637 +柚 1638 +柜 1639 +柠 1640 +查 1641 +柯 1642 +柱 1643 +柳 1644 +柴 1645 +柿 1646 +栀 1647 +栅 1648 +标 1649 +栈 1650 +栋 1651 +栏 1652 +树 1653 +栓 1654 +栖 1655 +栗 1656 +校 1657 +株 1658 +样 1659 +核 1660 +根 1661 +格 1662 +栽 1663 +桂 1664 +桃 1665 +框 1666 +案 1667 +桌 1668 +桐 1669 +桑 1670 +桔 1671 +桢 1672 +档 1673 +桥 1674 +桦 1675 +桨 1676 +桩 1677 +桶 1678 +梁 1679 +梅 1680 +梗 1681 +梦 1682 +梨 1683 +梭 1684 +梯 1685 +械 1686 +梳 1687 +检 1688 +棉 1689 +棋 1690 +棍 1691 +棒 1692 +棘 1693 +棚 1694 +棠 1695 +棣 1696 +森 1697 +棱 1698 +棵 1699 +椅 1700 +植 1701 +椎 1702 +椒 1703 +椭 1704 +椰 1705 +楂 1706 +楚 1707 +楠 1708 +楼 1709 +概 1710 +榄 1711 +榆 1712 +榕 1713 +榜 1714 +榨 1715 +榴 1716 +槐 1717 +槛 1718 +槟 1719 +槽 1720 +樊 1721 +樟 1722 +模 1723 +横 1724 +樱 1725 +橄 1726 +橇 1727 +橙 1728 +橱 1729 +檐 1730 +檞 1731 +檬 1732 +欠 1733 +次 1734 +欢 1735 +欣 1736 +欧 1737 +欲 1738 +欺 1739 +款 1740 +歇 1741 +歉 1742 +歌 1743 +止 1744 +正 1745 +此 1746 +步 1747 +武 1748 +歧 1749 +歪 1750 +歹 1751 +死 1752 +殊 1753 +残 1754 +殖 1755 +殴 1756 +段 1757 +殷 1758 +殿 1759 +毁 1760 +毅 1761 +母 1762 +每 1763 +毒 1764 +比 1765 +毕 1766 +毙 1767 +毛 1768 +毡 1769 +毫 1770 +毯 1771 +氏 1772 +民 1773 +氓 1774 +气 1775 +氛 1776 +氧 1777 +氮 1778 +水 1779 +永 1780 +汁 1781 +求 1782 +汇 1783 +汉 1784 +汗 1785 +江 1786 +池 1787 +污 1788 +汤 1789 +汪 1790 +汰 1791 +汹 1792 +汽 1793 +沃 1794 +沈 1795 +沉 1796 +沐 1797 +沓 1798 +沙 1799 +沛 1800 +沟 1801 +没 1802 +沥 1803 +沦 1804 +沧 1805 +沫 1806 +沮 1807 +河 1808 +沸 1809 +油 1810 +治 1811 +沼 1812 +沾 1813 +沿 1814 +泄 1815 +泉 1816 +泊 1817 +泌 1818 +法 1819 +泛 1820 +泞 1821 +泡 1822 +波 1823 +泥 1824 +注 1825 +泪 1826 +泯 1827 +泰 1828 +泱 1829 +泳 1830 +泸 1831 +泼 1832 +泽 1833 +洁 1834 +洋 1835 +洒 1836 +洗 1837 +洛 1838 +洞 1839 +津 1840 +洪 1841 +洲 1842 +活 1843 +洽 1844 +派 1845 +流 1846 +浅 1847 +浆 1848 +浇 1849 +浊 1850 +测 1851 +浍 1852 +济 1853 +浏 1854 +浑 1855 +浒 1856 +浓 1857 +浙 1858 +浜 1859 +浦 1860 +浩 1861 +浪 1862 +浮 1863 +浴 1864 +海 1865 +浸 1866 +涂 1867 +涅 1868 +消 1869 +涉 1870 +涌 1871 +涎 1872 +涕 1873 +涛 1874 +涝 1875 +涡 1876 +涣 1877 +润 1878 +涨 1879 +涮 1880 +涯 1881 +液 1882 +涵 1883 +淀 1884 +淆 1885 +淇 1886 +淋 1887 +淌 1888 +淑 1889 +淘 1890 +淞 1891 +淡 1892 +淫 1893 +淮 1894 +深 1895 +淳 1896 +混 1897 +淹 1898 +添 1899 +清 1900 +渊 1901 +渎 1902 +渐 1903 +渔 1904 +渗 1905 +渝 1906 +渠 1907 +渡 1908 +渣 1909 +渤 1910 +温 1911 +港 1912 +渲 1913 +渴 1914 +游 1915 +渺 1916 +湃 1917 +湖 1918 +湘 1919 +湛 1920 +湾 1921 +湿 1922 +溃 1923 +溅 1924 +源 1925 +溜 1926 +溢 1927 +溥 1928 +溪 1929 +溶 1930 +溺 1931 +滁 1932 +滋 1933 +滑 1934 +滔 1935 +滕 1936 +滚 1937 +滞 1938 +满 1939 +滤 1940 +滥 1941 +滨 1942 +滩 1943 +滴 1944 +漂 1945 +漆 1946 +漏 1947 +漓 1948 +演 1949 +漠 1950 +漫 1951 +漳 1952 +潆 1953 +潇 1954 +潍 1955 +潘 1956 +潜 1957 +潞 1958 +潢 1959 +潭 1960 +潮 1961 +潸 1962 +潼 1963 +澄 1964 +澈 1965 +澎 1966 +澜 1967 +澡 1968 +澳 1969 +激 1970 +濛 1971 +濡 1972 +濮 1973 +瀑 1974 +灌 1975 +火 1976 +灭 1977 +灯 1978 +灰 1979 +灵 1980 +灶 1981 +灼 1982 +灾 1983 +灿 1984 +炀 1985 +炉 1986 +炊 1987 +炎 1988 +炒 1989 +炕 1990 +炖 1991 +炝 1992 +炫 1993 +炬 1994 +炭 1995 +炮 1996 +炯 1997 +炳 1998 +炸 1999 +点 2000 +炼 2001 +烁 2002 +烂 2003 +烈 2004 +烘 2005 +烙 2006 +烛 2007 +烟 2008 +烤 2009 +烦 2010 +烧 2011 +烨 2012 +烩 2013 +烫 2014 +热 2015 +烹 2016 +焉 2017 +焊 2018 +焕 2019 +焖 2020 +焚 2021 +焦 2022 +焰 2023 +然 2024 +煅 2025 +煌 2026 +煎 2027 +煜 2028 +煞 2029 +煤 2030 +照 2031 +煨 2032 +煮 2033 +煲 2034 +煸 2035 +煽 2036 +熄 2037 +熊 2038 +熏 2039 +熔 2040 +熘 2041 +熙 2042 +熟 2043 +熬 2044 +燃 2045 +燎 2046 +燕 2047 +燥 2048 +爆 2049 +爪 2050 +爬 2051 +爱 2052 +爵 2053 +父 2054 +爷 2055 +爸 2056 +爹 2057 +爽 2058 +片 2059 +版 2060 +牌 2061 +牙 2062 +牛 2063 +牡 2064 +牢 2065 +牧 2066 +物 2067 +牲 2068 +牵 2069 +特 2070 +牺 2071 +犀 2072 +犄 2073 +犊 2074 +犟 2075 +犯 2076 +状 2077 +犷 2078 +犹 2079 +狂 2080 +狄 2081 +狈 2082 +狐 2083 +狗 2084 +狙 2085 +狞 2086 +狠 2087 +狡 2088 +独 2089 +狭 2090 +狮 2091 +狱 2092 +狸 2093 +狼 2094 +猎 2095 +猕 2096 +猖 2097 +猛 2098 +猜 2099 +猪 2100 +猫 2101 +猬 2102 +献 2103 +猴 2104 +獗 2105 +玄 2106 +率 2107 +玉 2108 +王 2109 +玛 2110 +玟 2111 +玩 2112 +玫 2113 +玮 2114 +环 2115 +现 2116 +玲 2117 +玻 2118 +珀 2119 +珈 2120 +珊 2121 +珍 2122 +珑 2123 +珠 2124 +班 2125 +球 2126 +理 2127 +琐 2128 +琢 2129 +琥 2130 +琦 2131 +琪 2132 +琳 2133 +琴 2134 +琵 2135 +琶 2136 +琼 2137 +瑕 2138 +瑜 2139 +瑞 2140 +瑟 2141 +瑰 2142 +瑶 2143 +璃 2144 +璇 2145 +璋 2146 +璐 2147 +璜 2148 +璧 2149 +瓜 2150 +瓣 2151 +瓦 2152 +瓶 2153 +瓷 2154 +甄 2155 +甑 2156 +甘 2157 +甚 2158 +甜 2159 +生 2160 +甥 2161 +用 2162 +甩 2163 +甫 2164 +甭 2165 +田 2166 +由 2167 +甲 2168 +申 2169 +电 2170 +男 2171 +画 2172 +畅 2173 +界 2174 +畏 2175 +畔 2176 +留 2177 +略 2178 +番 2179 +畴 2180 +畸 2181 +疆 2182 +疏 2183 +疑 2184 +疗 2185 +疙 2186 +疚 2187 +疡 2188 +疤 2189 +疫 2190 +疯 2191 +疲 2192 +疵 2193 +疼 2194 +疾 2195 +病 2196 +症 2197 +痒 2198 +痕 2199 +痘 2200 +痛 2201 +痞 2202 +痣 2203 +痰 2204 +痴 2205 +痿 2206 +瘠 2207 +瘤 2208 +瘦 2209 +瘩 2210 +瘴 2211 +瘾 2212 +癌 2213 +癖 2214 +癫 2215 +登 2216 +白 2217 +百 2218 +皂 2219 +的 2220 +皆 2221 +皇 2222 +皑 2223 +皓 2224 +皖 2225 +皮 2226 +皱 2227 +盆 2228 +盈 2229 +益 2230 +盐 2231 +监 2232 +盒 2233 +盔 2234 +盖 2235 +盗 2236 +盘 2237 +盛 2238 +盟 2239 +目 2240 +盯 2241 +盲 2242 +直 2243 +相 2244 +盼 2245 +盾 2246 +省 2247 +眉 2248 +看 2249 +真 2250 +眠 2251 +眨 2252 +眩 2253 +眯 2254 +眶 2255 +眷 2256 +眺 2257 +眼 2258 +着 2259 +睁 2260 +睐 2261 +睛 2262 +睡 2263 +督 2264 +睦 2265 +睫 2266 +睹 2267 +瞄 2268 +瞅 2269 +瞌 2270 +瞎 2271 +瞒 2272 +瞟 2273 +瞧 2274 +瞩 2275 +瞪 2276 +瞬 2277 +瞻 2278 +瞿 2279 +矛 2280 +矜 2281 +矢 2282 +矣 2283 +知 2284 +矩 2285 +矫 2286 +短 2287 +矮 2288 +石 2289 +矶 2290 +矿 2291 +码 2292 +砂 2293 +砌 2294 +砍 2295 +砒 2296 +研 2297 +砖 2298 +砣 2299 +破 2300 +砸 2301 +础 2302 +硅 2303 +硕 2304 +硝 2305 +硫 2306 +硬 2307 +确 2308 +碌 2309 +碍 2310 +碎 2311 +碑 2312 +碗 2313 +碟 2314 +碧 2315 +碰 2316 +碳 2317 +碴 2318 +磁 2319 +磅 2320 +磊 2321 +磋 2322 +磕 2323 +磨 2324 +磷 2325 +礁 2326 +礴 2327 +示 2328 +礼 2329 +社 2330 +祁 2331 +祈 2332 +祖 2333 +祜 2334 +祝 2335 +神 2336 +祠 2337 +祥 2338 +票 2339 +祭 2340 +祷 2341 +祸 2342 +禁 2343 +禅 2344 +福 2345 +禧 2346 +禹 2347 +离 2348 +禾 2349 +秀 2350 +私 2351 +秃 2352 +秋 2353 +种 2354 +科 2355 +秒 2356 +秘 2357 +租 2358 +秤 2359 +秦 2360 +秧 2361 +秩 2362 +积 2363 +称 2364 +移 2365 +秽 2366 +稀 2367 +程 2368 +稍 2369 +税 2370 +稚 2371 +稠 2372 +稣 2373 +稳 2374 +稷 2375 +稻 2376 +稼 2377 +稽 2378 +稿 2379 +穆 2380 +穗 2381 +究 2382 +穷 2383 +空 2384 +穿 2385 +突 2386 +窃 2387 +窄 2388 +窈 2389 +窍 2390 +窑 2391 +窕 2392 +窗 2393 +窘 2394 +窜 2395 +窝 2396 +窟 2397 +窦 2398 +窿 2399 +立 2400 +竖 2401 +站 2402 +竞 2403 +竟 2404 +章 2405 +童 2406 +竭 2407 +端 2408 +竹 2409 +笈 2410 +笋 2411 +笑 2412 +笔 2413 +笛 2414 +符 2415 +笨 2416 +第 2417 +笼 2418 +等 2419 +筋 2420 +筐 2421 +筑 2422 +筒 2423 +答 2424 +策 2425 +筛 2426 +筝 2427 +筷 2428 +筹 2429 +签 2430 +简 2431 +箍 2432 +算 2433 +管 2434 +箫 2435 +箭 2436 +箱 2437 +篇 2438 +篝 2439 +篡 2440 +篮 2441 +篷 2442 +簧 2443 +簿 2444 +籁 2445 +籍 2446 +米 2447 +类 2448 +籽 2449 +粉 2450 +粒 2451 +粕 2452 +粗 2453 +粘 2454 +粟 2455 +粤 2456 +粥 2457 +粪 2458 +粮 2459 +粱 2460 +粹 2461 +精 2462 +糅 2463 +糊 2464 +糕 2465 +糖 2466 +糙 2467 +糜 2468 +糟 2469 +糯 2470 +系 2471 +紊 2472 +素 2473 +索 2474 +紧 2475 +紫 2476 +累 2477 +絮 2478 +繁 2479 +纠 2480 +红 2481 +纣 2482 +纤 2483 +约 2484 +级 2485 +纨 2486 +纪 2487 +纬 2488 +纭 2489 +纯 2490 +纱 2491 +纲 2492 +纳 2493 +纵 2494 +纷 2495 +纸 2496 +纹 2497 +纺 2498 +纽 2499 +线 2500 +练 2501 +组 2502 +绅 2503 +细 2504 +织 2505 +终 2506 +绊 2507 +绍 2508 +绎 2509 +经 2510 +绑 2511 +绒 2512 +结 2513 +绔 2514 +绕 2515 +绘 2516 +给 2517 +络 2518 +绝 2519 +绞 2520 +统 2521 +绢 2522 +绣 2523 +继 2524 +绩 2525 +绪 2526 +绫 2527 +续 2528 +绮 2529 +绯 2530 +绰 2531 +绳 2532 +维 2533 +绵 2534 +绷 2535 +绸 2536 +综 2537 +绽 2538 +绿 2539 +缀 2540 +缆 2541 +缇 2542 +缓 2543 +编 2544 +缘 2545 +缚 2546 +缝 2547 +缠 2548 +缤 2549 +缩 2550 +缴 2551 +缸 2552 +缺 2553 +罂 2554 +罐 2555 +网 2556 +罕 2557 +罗 2558 +罚 2559 +罢 2560 +罩 2561 +罪 2562 +置 2563 +署 2564 +羁 2565 +羊 2566 +美 2567 +羔 2568 +羚 2569 +羞 2570 +羡 2571 +群 2572 +羸 2573 +羹 2574 +羽 2575 +翁 2576 +翅 2577 +翔 2578 +翘 2579 +翠 2580 +翡 2581 +翩 2582 +翰 2583 +翱 2584 +翻 2585 +翼 2586 +耀 2587 +老 2588 +考 2589 +者 2590 +而 2591 +耍 2592 +耐 2593 +耕 2594 +耗 2595 +耳 2596 +耶 2597 +耸 2598 +耻 2599 +耽 2600 +聂 2601 +聊 2602 +聋 2603 +职 2604 +联 2605 +聘 2606 +聚 2607 +聪 2608 +肃 2609 +肆 2610 +肇 2611 +肉 2612 +肋 2613 +肌 2614 +肖 2615 +肘 2616 +肚 2617 +肝 2618 +肠 2619 +股 2620 +肢 2621 +肤 2622 +肥 2623 +肩 2624 +肪 2625 +肮 2626 +肯 2627 +育 2628 +肴 2629 +肺 2630 +肾 2631 +肿 2632 +胀 2633 +胁 2634 +胃 2635 +胆 2636 +背 2637 +胎 2638 +胖 2639 +胜 2640 +胞 2641 +胡 2642 +胤 2643 +胧 2644 +胫 2645 +胭 2646 +胳 2647 +胶 2648 +胸 2649 +能 2650 +脂 2651 +脆 2652 +脉 2653 +脊 2654 +脏 2655 +脐 2656 +脑 2657 +脖 2658 +脚 2659 +脯 2660 +脱 2661 +脸 2662 +脾 2663 +腆 2664 +腊 2665 +腋 2666 +腌 2667 +腐 2668 +腑 2669 +腓 2670 +腔 2671 +腕 2672 +腥 2673 +腩 2674 +腰 2675 +腹 2676 +腺 2677 +腻 2678 +腼 2679 +腾 2680 +腿 2681 +膀 2682 +膊 2683 +膏 2684 +膜 2685 +膝 2686 +膨 2687 +膳 2688 +膻 2689 +臀 2690 +臂 2691 +臃 2692 +臣 2693 +臧 2694 +自 2695 +臭 2696 +至 2697 +致 2698 +舅 2699 +舆 2700 +舌 2701 +舍 2702 +舒 2703 +舔 2704 +舜 2705 +舞 2706 +舟 2707 +航 2708 +般 2709 +舰 2710 +舵 2711 +船 2712 +艇 2713 +艘 2714 +良 2715 +艰 2716 +色 2717 +艳 2718 +艺 2719 +艾 2720 +节 2721 +芋 2722 +芒 2723 +芙 2724 +芜 2725 +芝 2726 +芦 2727 +芬 2728 +芭 2729 +芮 2730 +花 2731 +芳 2732 +芸 2733 +芹 2734 +芽 2735 +苇 2736 +苍 2737 +苏 2738 +苔 2739 +苗 2740 +苛 2741 +苞 2742 +苟 2743 +若 2744 +苦 2745 +英 2746 +苹 2747 +茂 2748 +范 2749 +茄 2750 +茅 2751 +茉 2752 +茜 2753 +茧 2754 +茨 2755 +茫 2756 +茱 2757 +茵 2758 +茶 2759 +茸 2760 +茹 2761 +荆 2762 +草 2763 +荐 2764 +荒 2765 +荔 2766 +荞 2767 +荠 2768 +荡 2769 +荣 2770 +荤 2771 +荦 2772 +荫 2773 +荮 2774 +药 2775 +荷 2776 +荼 2777 +莉 2778 +莎 2779 +莓 2780 +莞 2781 +莫 2782 +莱 2783 +莲 2784 +获 2785 +莹 2786 +莺 2787 +菇 2788 +菊 2789 +菌 2790 +菜 2791 +菠 2792 +菩 2793 +菱 2794 +菲 2795 +萃 2796 +萄 2797 +萌 2798 +萍 2799 +萎 2800 +萝 2801 +营 2802 +萧 2803 +萨 2804 +萱 2805 +落 2806 +著 2807 +葛 2808 +葡 2809 +董 2810 +葫 2811 +葬 2812 +葱 2813 +葳 2814 +葵 2815 +蒂 2816 +蒋 2817 +蒙 2818 +蒜 2819 +蒲 2820 +蒸 2821 +蓄 2822 +蓉 2823 +蓓 2824 +蓝 2825 +蓬 2826 +蔓 2827 +蔗 2828 +蔚 2829 +蔡 2830 +蔬 2831 +蔷 2832 +蔼 2833 +蔽 2834 +蕃 2835 +蕉 2836 +蕊 2837 +蕙 2838 +蕴 2839 +蕾 2840 +薄 2841 +薇 2842 +薛 2843 +薪 2844 +薯 2845 +藉 2846 +藏 2847 +藕 2848 +藤 2849 +藩 2850 +蘑 2851 +蘸 2852 +虎 2853 +虏 2854 +虐 2855 +虑 2856 +虔 2857 +虚 2858 +虞 2859 +虫 2860 +虱 2861 +虹 2862 +虻 2863 +虽 2864 +虾 2865 +蚀 2866 +蚁 2867 +蚂 2868 +蚊 2869 +蚌 2870 +蚓 2871 +蚕 2872 +蚯 2873 +蛀 2874 +蛆 2875 +蛇 2876 +蛋 2877 +蛙 2878 +蛛 2879 +蛮 2880 +蜀 2881 +蜂 2882 +蜇 2883 +蜒 2884 +蜓 2885 +蜗 2886 +蜘 2887 +蜚 2888 +蜜 2889 +蜡 2890 +蜻 2891 +蝇 2892 +蝉 2893 +蝎 2894 +蝗 2895 +蝙 2896 +蝠 2897 +蝴 2898 +蝶 2899 +螃 2900 +融 2901 +螺 2902 +蟹 2903 +蠢 2904 +血 2905 +衅 2906 +行 2907 +衍 2908 +衔 2909 +街 2910 +衙 2911 +衡 2912 +衣 2913 +补 2914 +表 2915 +衩 2916 +衫 2917 +衬 2918 +衮 2919 +衰 2920 +衷 2921 +袁 2922 +袂 2923 +袋 2924 +袍 2925 +袖 2926 +袜 2927 +袢 2928 +被 2929 +袭 2930 +袱 2931 +裁 2932 +裂 2933 +装 2934 +裕 2935 +裘 2936 +裙 2937 +裤 2938 +裨 2939 +裳 2940 +裴 2941 +裸 2942 +裹 2943 +褂 2944 +褒 2945 +褚 2946 +褶 2947 +襄 2948 +西 2949 +要 2950 +覆 2951 +见 2952 +观 2953 +规 2954 +觅 2955 +视 2956 +览 2957 +觉 2958 +角 2959 +解 2960 +触 2961 +言 2962 +詹 2963 +誉 2964 +誓 2965 +警 2966 +譬 2967 +计 2968 +订 2969 +认 2970 +讨 2971 +让 2972 +训 2973 +议 2974 +讯 2975 +记 2976 +讲 2977 +讳 2978 +讶 2979 +讷 2980 +许 2981 +讹 2982 +论 2983 +讽 2984 +设 2985 +访 2986 +诀 2987 +证 2988 +评 2989 +识 2990 +诈 2991 +诉 2992 +词 2993 +诏 2994 +译 2995 +试 2996 +诗 2997 +诙 2998 +诚 2999 +诛 3000 +话 3001 +诞 3002 +询 3003 +诣 3004 +该 3005 +详 3006 +诧 3007 +诬 3008 +语 3009 +误 3010 +诱 3011 +说 3012 +诵 3013 +诶 3014 +请 3015 +诸 3016 +诺 3017 +读 3018 +诽 3019 +课 3020 +谁 3021 +调 3022 +谅 3023 +谆 3024 +谈 3025 +谊 3026 +谋 3027 +谍 3028 +谎 3029 +谏 3030 +谐 3031 +谓 3032 +谕 3033 +谚 3034 +谛 3035 +谜 3036 +谢 3037 +谣 3038 +谤 3039 +谦 3040 +谨 3041 +谬 3042 +谭 3043 +谱 3044 +谴 3045 +谷 3046 +豁 3047 +豆 3048 +豇 3049 +豚 3050 +象 3051 +豪 3052 +豫 3053 +豹 3054 +豺 3055 +貂 3056 +貌 3057 +贝 3058 +贞 3059 +负 3060 +贡 3061 +财 3062 +责 3063 +贤 3064 +败 3065 +货 3066 +质 3067 +贩 3068 +贪 3069 +贫 3070 +贬 3071 +购 3072 +贯 3073 +贱 3074 +贴 3075 +贵 3076 +贷 3077 +贸 3078 +费 3079 +贺 3080 +贼 3081 +贾 3082 +贿 3083 +赂 3084 +资 3085 +赋 3086 +赌 3087 +赎 3088 +赏 3089 +赐 3090 +赔 3091 +赖 3092 +赘 3093 +赚 3094 +赛 3095 +赞 3096 +赠 3097 +赡 3098 +赢 3099 +赤 3100 +赦 3101 +赫 3102 +走 3103 +赴 3104 +赵 3105 +赶 3106 +起 3107 +趁 3108 +超 3109 +越 3110 +趋 3111 +趟 3112 +趣 3113 +足 3114 +趴 3115 +趾 3116 +跃 3117 +跆 3118 +跋 3119 +跌 3120 +跎 3121 +跑 3122 +距 3123 +跟 3124 +跤 3125 +跨 3126 +跪 3127 +路 3128 +跳 3129 +践 3130 +跷 3131 +跺 3132 +踏 3133 +踢 3134 +踩 3135 +踪 3136 +踮 3137 +踹 3138 +蹄 3139 +蹈 3140 +蹉 3141 +蹋 3142 +蹦 3143 +蹬 3144 +蹭 3145 +蹲 3146 +蹿 3147 +躁 3148 +身 3149 +躯 3150 +躲 3151 +躺 3152 +车 3153 +轧 3154 +轨 3155 +轩 3156 +转 3157 +轭 3158 +轮 3159 +软 3160 +轰 3161 +轱 3162 +轻 3163 +载 3164 +轿 3165 +较 3166 +辄 3167 +辅 3168 +辆 3169 +辈 3170 +辉 3171 +辐 3172 +辑 3173 +输 3174 +辘 3175 +辙 3176 +辛 3177 +辜 3178 +辞 3179 +辟 3180 +辣 3181 +辨 3182 +辩 3183 +辫 3184 +辰 3185 +辱 3186 +边 3187 +辽 3188 +达 3189 +迁 3190 +迄 3191 +迅 3192 +过 3193 +迈 3194 +迎 3195 +运 3196 +近 3197 +返 3198 +还 3199 +这 3200 +进 3201 +远 3202 +违 3203 +连 3204 +迟 3205 +迦 3206 +迪 3207 +迫 3208 +述 3209 +迷 3210 +迹 3211 +追 3212 +退 3213 +送 3214 +适 3215 +逃 3216 +逆 3217 +选 3218 +逊 3219 +逍 3220 +透 3221 +逐 3222 +递 3223 +途 3224 +逗 3225 +通 3226 +逛 3227 +逝 3228 +逞 3229 +速 3230 +造 3231 +逢 3232 +逮 3233 +逸 3234 +逻 3235 +逼 3236 +遇 3237 +遍 3238 +遏 3239 +道 3240 +遗 3241 +遛 3242 +遢 3243 +遣 3244 +遥 3245 +遭 3246 +遮 3247 +遵 3248 +避 3249 +邀 3250 +邋 3251 +邓 3252 +邢 3253 +那 3254 +邦 3255 +邪 3256 +邮 3257 +邯 3258 +邰 3259 +邱 3260 +邳 3261 +邵 3262 +邻 3263 +郁 3264 +郅 3265 +郊 3266 +郎 3267 +郑 3268 +郝 3269 +郡 3270 +郦 3271 +部 3272 +郭 3273 +都 3274 +鄙 3275 +鄱 3276 +酋 3277 +配 3278 +酒 3279 +酗 3280 +酝 3281 +酣 3282 +酥 3283 +酬 3284 +酱 3285 +酷 3286 +酸 3287 +酿 3288 +醇 3289 +醉 3290 +醋 3291 +醒 3292 +采 3293 +释 3294 +里 3295 +重 3296 +野 3297 +量 3298 +金 3299 +釜 3300 +鉴 3301 +鑫 3302 +针 3303 +钉 3304 +钓 3305 +钗 3306 +钙 3307 +钛 3308 +钝 3309 +钞 3310 +钟 3311 +钠 3312 +钡 3313 +钢 3314 +钥 3315 +钦 3316 +钩 3317 +钱 3318 +钻 3319 +铁 3320 +铃 3321 +铅 3322 +铎 3323 +铛 3324 +铜 3325 +铭 3326 +铮 3327 +铲 3328 +银 3329 +铸 3330 +铺 3331 +链 3332 +铿 3333 +销 3334 +锁 3335 +锄 3336 +锅 3337 +锈 3338 +锉 3339 +锋 3340 +锌 3341 +锐 3342 +锔 3343 +错 3344 +锚 3345 +锡 3346 +锣 3347 +锤 3348 +锦 3349 +键 3350 +锵 3351 +锹 3352 +锻 3353 +镀 3354 +镇 3355 +镐 3356 +镑 3357 +镕 3358 +镖 3359 +镜 3360 +镭 3361 +镯 3362 +镶 3363 +长 3364 +门 3365 +闪 3366 +闭 3367 +问 3368 +闯 3369 +闰 3370 +闲 3371 +间 3372 +闷 3373 +闸 3374 +闹 3375 +闺 3376 +闻 3377 +阀 3378 +阁 3379 +阂 3380 +阅 3381 +阎 3382 +阐 3383 +阔 3384 +阜 3385 +队 3386 +阱 3387 +防 3388 +阳 3389 +阴 3390 +阵 3391 +阶 3392 +阻 3393 +阿 3394 +陀 3395 +附 3396 +际 3397 +陆 3398 +陈 3399 +陋 3400 +陌 3401 +降 3402 +限 3403 +陕 3404 +陡 3405 +院 3406 +除 3407 +陨 3408 +险 3409 +陪 3410 +陵 3411 +陶 3412 +陷 3413 +隆 3414 +隋 3415 +隍 3416 +随 3417 +隐 3418 +隔 3419 +隘 3420 +障 3421 +隧 3422 +隶 3423 +难 3424 +雀 3425 +雁 3426 +雄 3427 +雅 3428 +集 3429 +雇 3430 +雍 3431 +雕 3432 +雨 3433 +雪 3434 +雯 3435 +零 3436 +雷 3437 +雾 3438 +需 3439 +霄 3440 +霆 3441 +震 3442 +霉 3443 +霍 3444 +霎 3445 +霏 3446 +霓 3447 +霜 3448 +霞 3449 +露 3450 +霸 3451 +青 3452 +靓 3453 +靖 3454 +静 3455 +非 3456 +靠 3457 +靡 3458 +面 3459 +革 3460 +靴 3461 +靶 3462 +鞋 3463 +鞍 3464 +鞭 3465 +韦 3466 +韧 3467 +韩 3468 +韭 3469 +音 3470 +韵 3471 +韶 3472 +页 3473 +顶 3474 +项 3475 +顺 3476 +须 3477 +顽 3478 +顾 3479 +顿 3480 +颁 3481 +颂 3482 +预 3483 +领 3484 +颇 3485 +颈 3486 +颊 3487 +颐 3488 +频 3489 +颓 3490 +颖 3491 +颗 3492 +题 3493 +颜 3494 +额 3495 +颠 3496 +颤 3497 +颦 3498 +风 3499 +飕 3500 +飘 3501 +飙 3502 +飞 3503 +食 3504 +餐 3505 +餮 3506 +饥 3507 +饪 3508 +饭 3509 +饮 3510 +饰 3511 +饱 3512 +饶 3513 +饺 3514 +饼 3515 +饿 3516 +馄 3517 +馅 3518 +馆 3519 +馈 3520 +馋 3521 +馍 3522 +馒 3523 +首 3524 +香 3525 +馨 3526 +马 3527 +驭 3528 +驮 3529 +驰 3530 +驱 3531 +驴 3532 +驶 3533 +驸 3534 +驹 3535 +驻 3536 +驼 3537 +驾 3538 +驿 3539 +骂 3540 +骄 3541 +骆 3542 +骋 3543 +验 3544 +骏 3545 +骑 3546 +骗 3547 +骚 3548 +骛 3549 +骞 3550 +骤 3551 +骨 3552 +骷 3553 +骸 3554 +骼 3555 +髅 3556 +髓 3557 +高 3558 +髦 3559 +鬼 3560 +魁 3561 +魂 3562 +魄 3563 +魅 3564 +魏 3565 +魔 3566 +鱼 3567 +鱿 3568 +鲁 3569 +鲍 3570 +鲜 3571 +鲤 3572 +鲨 3573 +鲫 3574 +鲸 3575 +鳅 3576 +鳌 3577 +鳖 3578 +鳝 3579 +鳞 3580 +鸟 3581 +鸡 3582 +鸣 3583 +鸦 3584 +鸭 3585 +鸯 3586 +鸳 3587 +鸽 3588 +鸿 3589 +鹅 3590 +鹏 3591 +鹤 3592 +鹰 3593 +鹿 3594 +麋 3595 +麒 3596 +麟 3597 +麦 3598 +麻 3599 +麽 3600 +黄 3601 +黎 3602 +黏 3603 +黑 3604 +默 3605 +黛 3606 +黝 3607 +黟 3608 +黯 3609 +鼎 3610 +鼓 3611 +鼠 3612 +鼻 3613 +齐 3614 +齿 3615 +龄 3616 +龊 3617 +龌 3618 +龙 3619 +龚 3620 +龟 3621 +＋ 3622 +a 3623 +b 3624 +c 3625 +d 3626 +e 3627 +f 3628 +g 3629 +h 3630 +i 3631 +j 3632 +k 3633 +l 3634 +m 3635 +n 3636 +o 3637 +p 3638 +q 3639 +r 3640 +s 3641 +t 3642 +u 3643 +v 3644 +w 3645 +x 3646 +y 3647 +z 3648 + 3649 diff --git a/examples/asr/hkust/deep_speech.json b/examples/asr/hkust/deep_speech.json new file mode 100644 index 00000000..23984bad --- /dev/null +++ b/examples/asr/hkust/deep_speech.json @@ -0,0 +1,47 @@ +{ + "batch_size":32, + "num_epochs":20, + "sorta_epoch":1, + "ckpt":"examples/asr/hkust/ckpts/deep_speech", + "solver_gpu":[2], + "solver_config":{ + "clip_norm":100.0, + "log_interval":1 + }, + + "model":"deep_speech", + "num_classes": null, + "pretrained_model": null, + "model_config":{ + "conv_filters":64, + "rnn_hidden_size":1680, + "rnn_type":"cudnngru", + "num_rnn_layers":6 + }, + + "optimizer":"warmup_adam", + "optimizer_config":{ + "d_model":512, + "warmup_steps":8000, + "k":0.5 + }, + + "dataset_builder": "speech_recognition_dataset", + "dataset_config":{ + "audio_config":{ + "type":"Fbank", + "filterbank_channel_count":40, + "local_cmvn":false + }, + "cmvn_file":"examples/asr/hkust/data/cmvn", + "text_config": { + "type":"vocab", + "model":"examples/asr/hkust/data/vocab" + }, + "input_length_range":[10, 8000] + }, + "num_data_threads": 1, + "train_csv":"examples/asr/hkust/data/train.csv", + "dev_csv":"examples/asr/hkust/data/dev.csv", + "test_csv":"examples/asr/hkust/data/dev.csv" +} \ No newline at end of file diff --git a/examples/asr/hkust/local/prepare_data.py b/examples/asr/hkust/local/prepare_data.py new file mode 100644 index 00000000..46efdbff --- /dev/null +++ b/examples/asr/hkust/local/prepare_data.py @@ -0,0 +1,203 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=invalid-name +""" hkust dataset """ + +import os +import re +import sys +import codecs +from tempfile import TemporaryDirectory +import fnmatch +import pandas +from absl import logging + +import tensorflow as tf +from sox import Transformer +from athena import get_wave_file_length +from athena import TextFeaturizer + +SUBSETS = ["train", "dev"] + + +def normalize_hkust_trans(trans): + """ normalize HKUST transcripts + delete unuseful symbols and keep space between English words + Args: + trans: original transcripts + """ + norm_trans = re.sub(r"<.*?>|{.*?}", r"", trans) + tmp_trans = "" + for item in norm_trans.strip().split(): + if re.search("[a-zA-Z]", item): + item = " " + item + " " + tmp_trans += item + norm_trans = re.sub(r"\s{2,}", r" ", tmp_trans) + norm_trans = norm_trans.strip() + return norm_trans + + +def convert_audio_and_split_transcript(directory, subset, out_csv_file): + """Convert SPH to WAV and split the transcript. + + Args: + directory: the directory which holds the input dataset. + subset: the name of the specified dataset. e.g. dev + out_csv_file: the resulting output csv file + """ + gfile = tf.compat.v1.gfile + sph2pip = os.path.join(os.path.dirname(__file__), "../../../../athena/tools/sph2pipe") + text_featurizer = TextFeaturizer() + + logging.info("Processing audio and transcript for %s" % subset) + audio_dir = os.path.join(directory, "LDC2005S15/") + trans_dir = os.path.join(directory, "LDC2005T32/") + + output_wav_dir = os.path.join(directory, subset + "/wav") + if not gfile.Exists(output_wav_dir): + gfile.MakeDirs(output_wav_dir) + + files = [] + char_dict = {} + + sph_files_dict = {} + for root, _, filenames in gfile.Walk(audio_dir): + for filename in fnmatch.filter(filenames, "*.sph"): + if subset in root: + sph_key = os.path.splitext(filename)[0] + sph_file = os.path.join(root, filename) + sph_files_dict[sph_key] = sph_file + + # Convert all SPH file into WAV format. + # Generate the JSON file and char dict file. + with TemporaryDirectory(dir="/tmp-data/tmp/") as output_tmp_wav_dir: + for root, _, filenames in gfile.Walk(trans_dir): + if not re.match('.*/' + subset + '.*', root): + continue + for filename in fnmatch.filter(filenames, "*.txt"): + trans_file = os.path.join(root, filename) + sph_key = "" + speaker_A = "" + speaker_B = "" + with codecs.open(trans_file, "r", "gb18030") as fin: + for line in fin: + line = line.strip() + if len(line.split(" ")) <= 1: + continue + if len(line.split(" ")) == 2: + sph_key = line.split(" ")[1] + speaker_A = sph_key.split("_")[2] + speaker_B = sph_key.split("_")[3] + continue + + time_start, time_end, speaker, transcript = line.split(" ", 3) + time_start = float(time_start) + time_end = float(time_end) + # too short, skip the wave file + if time_end - time_start <= 0.1: + continue + + speaker = speaker[0] # remove ':' in 'A:' + if speaker == "A": + channel = 1 + speaker_id = speaker_A + else: + channel = 2 + speaker_id = speaker_B + + # Convert SPH to split WAV. + sph_file = sph_files_dict[sph_key] + wav_file = os.path.join( + output_tmp_wav_dir, sph_key + "." + speaker[0] + ".wav" + ) + if not gfile.Exists(sph_file): + raise ValueError( + "the sph file {} is not exists".format(sph_file) + ) + if not gfile.Exists(wav_file): + sph2pipe_cmd = ( + sph2pip + + " -f wav -c {} -p ".format(str(channel)) + + sph_file + + " " + + wav_file + ) + os.system(sph2pipe_cmd) + + sub_wav_filename = "{0}-{1}-{2:06d}-{3:06d}".format( + sph_key, speaker, int(time_start * 100), int(time_end * 100) + ) + sub_wav_file = os.path.join( + output_wav_dir, sub_wav_filename + ".wav" + ) + if not gfile.Exists(sub_wav_file): + tfm = Transformer() + tfm.trim(time_start, time_end) + tfm.build(wav_file, sub_wav_file) + + wav_length = get_wave_file_length(sub_wav_file) + + transcript = normalize_hkust_trans(transcript) + transcript = text_featurizer.delete_punct(transcript) + + if len(transcript) > 0: + for char in transcript: + if char in char_dict: + char_dict[char] += 1 + else: + char_dict[char] = 0 + files.append( + ( + os.path.abspath(sub_wav_file), + wav_length, + transcript, + speaker_id, + ) + ) + + # Write to CSV file which contains three columns: + # "wav_filename", "wav_length_ms", "labels". + df = pandas.DataFrame( + data=files, columns=["wav_filename", "wav_length_ms", "transcript", "speaker"] + ) + df.to_csv(out_csv_file, index=False, sep="\t") + logging.info("Successfully generated csv file {}".format(out_csv_file)) + + +def processor(dircetory, subset, force_process): + """ download and process """ + if subset not in SUBSETS: + raise ValueError(subset, "is not in HKUST") + + subset_csv = os.path.join(dircetory, subset + ".csv") + if not force_process and os.path.exists(subset_csv): + return subset_csv + logging.info("Processing the HKUST subset {} in {}".format(subset, dircetory)) + convert_audio_and_split_transcript(dircetory, subset, subset_csv) + logging.info("Finished processing HKUST subset {}".format(subset)) + return subset_csv + + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + if len(sys.argv) < 2: + print('Usage: python {} data_dir (data_dir should contain audio and text ' + 'directory: LDC2005S15 and LDC2005T32)'.format(sys.argv[0])) + exit(1) + DIR = sys.argv[1] + for SUBSET in SUBSETS: + processor(DIR, SUBSET, True) diff --git a/examples/asr/hkust/local/segment_word.py b/examples/asr/hkust/local/segment_word.py new file mode 100644 index 00000000..46f2a453 --- /dev/null +++ b/examples/asr/hkust/local/segment_word.py @@ -0,0 +1,52 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" segment word """ + +import codecs +import sys +import re +from absl import logging +import jieba + + +def segment_trans(vocab_file, text_file): + ''' segment transcripts according to vocab + using Maximum Matching Algorithm + Args: + vocab_file: vocab file + text_file: transcripts file + Returns: + seg_trans: segment words + ''' + jieba.set_dictionary(vocab_file) + with open(text_file, "r", encoding="utf-8") as text: + lines = text.readlines() + sents = '' + for line in lines: + seg_line = jieba.cut(line.strip(), HMM=False) + seg_line = ' '.join(seg_line) + sents += seg_line + '\n' + return sents + + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach()) + if len(sys.argv) < 3: + logging.warning('Usage: python {} vocab_file text_file'.format(sys.argv[0])) + sys.exit() + print(segment_trans(sys.argv[1], sys.argv[2])) diff --git a/examples/asr/hkust/mpc.json b/examples/asr/hkust/mpc.json new file mode 100644 index 00000000..d863b12d --- /dev/null +++ b/examples/asr/hkust/mpc.json @@ -0,0 +1,47 @@ +{ + "batch_size":64, + "num_epochs":100, + "sorta_epoch":1, + "ckpt":"examples/asr/hkust/ckpts/mpc", + "solver_gpu":[0], + "solver_config":{ + "clip_norm":100, + "log_interval":10, + "enable_tf_function":true + }, + + "model": "mpc", + "num_classes": 40, + "model_config":{ + "return_encoder_output":false, + "num_filters":512, + "d_model":512, + "num_heads":8, + "num_encoder_layers":12, + "dff":1280, + "rate":0.1, + "chunk_size":1, + "keep_probability":0.8 + }, + + "optimizer": "warmup_adam", + "optimizer_config":{ + "d_model":512, + "warmup_steps":2000, + "k":0.3 + }, + + "dataset_builder":"speech_dataset", + "dataset_config":{ + "audio_config":{ + "type":"Fbank", + "filterbank_channel_count":40, + "local_cmvn":false + }, + "cmvn_file":"examples/asr/hkust/data/cmvn", + "input_length_range":[10, 8000] + }, + "num_data_threads": 1, + "train_csv":"examples/asr/hkust/data/train.csv", + "dev_csv":"examples/asr/hkust/data/dev.csv" +} diff --git a/examples/asr/hkust/mtl_transformer.json b/examples/asr/hkust/mtl_transformer.json new file mode 100644 index 00000000..911d6fd7 --- /dev/null +++ b/examples/asr/hkust/mtl_transformer.json @@ -0,0 +1,70 @@ +{ + "batch_size":32, + "num_epochs":13, + "sorta_epoch":2, + "ckpt":"examples/asr/hkust/ckpts/mtl_transformer_ctc/", + "summary_dir":"examples/asr/hkust/ckpts/mtl_transformer_ctc/event", + + "solver_gpu":[0], + "solver_config":{ + "clip_norm":100, + "log_interval":10, + "enable_tf_function":true + }, + + "model":"mtl_transformer_ctc", + "num_classes": null, + "pretrained_model": null, + "model_config":{ + "model":"speech_transformer", + "model_config":{ + "return_encoder_output":true, + "num_filters":512, + "d_model":512, + "num_heads":8, + "num_encoder_layers":12, + "num_decoder_layers":6, + "dff":1280, + "rate":0.1, + "label_smoothing_rate":0.0, + "schedual_sampling_rate":0.9 + }, + "mtl_weight":0.5 + }, + + "decode_config":{ + "beam_search":true, + "beam_size":4, + "ctc_weight":0.3, + "lm_weight":0.1, + "lm_path":"examples/asr/hkust/data/4gram.arpa" + }, + + "optimizer":"warmup_adam", + "optimizer_config":{ + "d_model":512, + "warmup_steps":8000, + "k":0.5, + "decay_steps": 50000, + "decay_rate": 0.1 + }, + + "dataset_builder": "speech_recognition_dataset", + "dataset_config":{ + "audio_config":{ + "type":"Fbank", + "filterbank_channel_count":40, + "local_cmvn":false + }, + "cmvn_file":"examples/asr/hkust/data/cmvn", + "text_config": { + "type":"vocab", + "model":"examples/asr/hkust/data/vocab" + }, + "input_length_range":[10, 8000] + }, + "num_data_threads": 1, + "train_csv":"examples/asr/hkust/data/train.csv", + "dev_csv":"examples/asr/hkust/data/dev.csv", + "test_csv":"examples/asr/hkust/data/dev.csv" +} diff --git a/examples/asr/hkust/mtl_transformer_sp.json b/examples/asr/hkust/mtl_transformer_sp.json new file mode 100644 index 00000000..113d3bdc --- /dev/null +++ b/examples/asr/hkust/mtl_transformer_sp.json @@ -0,0 +1,71 @@ +{ + "batch_size":32, + "num_epochs":15, + "sorta_epoch":2, + "ckpt":"examples/asr/hkust/ckpts/mtl_transformer_ctc_sp/", + "summary_dir":"examples/asr/hkust/ckpts/mtl_transformer_ctc_sp/event", + + "solver_gpu":[0], + "solver_config":{ + "clip_norm":100, + "log_interval":10, + "enable_tf_function":true + }, + + "model":"mtl_transformer_ctc", + "num_classes": null, + "pretrained_model": null, + "model_config":{ + "model":"speech_transformer", + "model_config":{ + "return_encoder_output":true, + "num_filters":512, + "d_model":512, + "num_heads":8, + "num_encoder_layers":12, + "num_decoder_layers":6, + "dff":1280, + "rate":0.1, + "label_smoothing_rate":0.0, + "schedual_sampling_rate":0.9 + }, + "mtl_weight":0.5 + }, + + "decode_config":{ + "beam_search":true, + "beam_size":4, + "ctc_weight":0.3, + "lm_weight":0.1, + "lm_path":"examples/asr/hkust/data/4gram.arpa" + }, + + "optimizer":"warmup_adam", + "optimizer_config":{ + "d_model":512, + "warmup_steps":8000, + "k":0.5, + "decay_steps": 140000, + "decay_rate": 0.1 + }, + + "dataset_builder": "speech_recognition_dataset", + "dataset_config":{ + "audio_config":{ + "type":"Fbank", + "filterbank_channel_count":40, + "local_cmvn":false + }, + "cmvn_file":"examples/asr/hkust/data/cmvn", + "text_config": { + "type":"vocab", + "model":"examples/asr/hkust/data/vocab" + }, + "input_length_range":[10, 8000], + "speed_permutation": [0.9, 1.0, 1.1] + }, + "num_data_threads": 1, + "train_csv":"examples/asr/hkust/data/train.csv", + "dev_csv":"examples/asr/hkust/data/dev.csv", + "test_csv":"examples/asr/hkust/data/dev.csv" +} diff --git a/examples/asr/hkust/rnnlm.json b/examples/asr/hkust/rnnlm.json new file mode 100644 index 00000000..d15796a0 --- /dev/null +++ b/examples/asr/hkust/rnnlm.json @@ -0,0 +1,49 @@ +{ + "batch_size":32, + "num_epochs":100, + "sorta_epoch":0, + "ckpt":"examples/asr/hkust/ckpts/rnnlm", + + "solver_gpu":[0], + "solver_config":{ + "clip_norm":100, + "log_interval":10, + "enable_tf_function":true + }, + + + "model":"rnnlm", + "num_classes": null, + "pretrained_model": null, + "model_config":{ + "d_model": 512, + "rnn_type": "lstm", + "num_layer": 4, + "dropout_rate": 0.1, + "sos": -1, + "eos": -1 + }, + + "optimizer":"warmup_adam", + "optimizer_config":{ + "d_model":512, + "warmup_steps":8000, + "k":0.5 + }, + + "dataset_builder": "language_dataset", + "dataset_config":{ + "input_text_config":{ + "type":"vocab", + "model":"examples/asr/hkust/data/vocab" + }, + "output_text_config":{ + "type":"vocab", + "model":"examples/asr/hkust/data/vocab" + } + }, + "num_data_threads": 1, + "train_csv":"examples/asr/hkust/data/train.trans.csv", + "dev_csv":"examples/asr/hkust/data/dev.trans.csv", + "test_csv":"examples/asr/hkust/data/dev.trans.csv" +} \ No newline at end of file diff --git a/examples/asr/hkust/run.sh b/examples/asr/hkust/run.sh new file mode 100644 index 00000000..bf9c9c8b --- /dev/null +++ b/examples/asr/hkust/run.sh @@ -0,0 +1,71 @@ +# coding=utf-8 +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +if [ "athena" != $(basename "$PWD") ]; then + echo "You should run this script in athena directory!!" + exit 1 +fi + +source tools/env.sh + +stage=0 +stop_stage=100 + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + # prepare data + echo "Preparing data" + python examples/asr/hkust/local/prepare_data.py /nfs/project/datasets/opensource_data/hkust + mkdir -p examples/asr/hkust/data + cp /nfs/project/datasets/opensource_data/hkust/{train,dev}.csv examples/asr/hkust/data/ + + # cal cmvn + cat examples/asr/hkust/data/train.csv > examples/asr/hkust/data/all.csv + tail -n +2 examples/asr/hkust/data/dev.csv >> examples/asr/hkust/data/all.csv + python athena/cmvn_main.py examples/asr/hkust/mpc.json examples/asr/hkust/data/all.csv +fi + +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + # pretrain stage + echo "Pretraining" + python athena/main.py examples/asr/hkust/mpc.json +fi + +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + # finetuning stage + echo "Fine-tuning" + python athena/main.py examples/asr/hkust/mtl_transformer.json +fi + +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + # decoding stage + echo "Decoding" + # prepare language model + tail -n +2 examples/asr/hkust/data/train.csv | cut -f 3 > examples/asr/hkust/data/text + python examples/asr/hkust/local/segment_word.py examples/asr/hkust/data/vocab \ + examples/asr/hkust/data/text > examples/asr/hkust/data/text.seg + tools/kenlm/build/bin/lmplz -o 4 < examples/asr/hkust/data/text.seg > examples/asr/hkust/data/4gram.arpa + + python athena/decode_main.py examples/asr/hkust/mtl_transformer.json +fi + +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + echo "training rnnlm" + tail -n +2 examples/asr/hkust/data/train.csv | awk '{print $3"\t"$3}' > examples/asr/hkust/data/train.trans.csv + tail -n +2 examples/asr/hkust/data/dev.csv | awk '{print $3"\t"$3}' > examples/asr/hkust/data/dev.trans.csv + python athena/main.py examples/asr/hkust/rnnlm.json +fi + diff --git a/examples/asr/hkust/transformer.json b/examples/asr/hkust/transformer.json new file mode 100644 index 00000000..d7cce588 --- /dev/null +++ b/examples/asr/hkust/transformer.json @@ -0,0 +1,55 @@ +{ + "batch_size":32, + "num_epochs":20, + "sorta_epoch":1, + "ckpt":"examples/asr/hkust/ckpts/transformer", + + "solver_gpu":[0], + "solver_config":{ + "clip_norm":100, + "log_interval":10, + "enable_tf_function":true + }, + + + "model":"speech_transformer", + "num_classes": null, + "pretrained_model": null, + "model_config":{ + "return_encoder_output":false, + "num_filters":512, + "d_model":512, + "num_heads":8, + "num_encoder_layers":12, + "num_decoder_layers":6, + "dff":1280, + "rate":0.1, + "label_smoothing_rate":0.0 + }, + + "optimizer":"warmup_adam", + "optimizer_config":{ + "d_model":512, + "warmup_steps":8000, + "k":0.5 + }, + + "dataset_builder": "speech_recognition_dataset", + "dataset_config":{ + "audio_config":{ + "type":"Fbank", + "filterbank_channel_count":40, + "local_cmvn":false + }, + "text_config": { + "type":"vocab", + "model":"examples/asr/hkust/data/vocab" + }, + "cmvn_file":"examples/asr/hkust/data/cmvn", + "input_length_range":[10, 5000] + }, + "num_data_threads": 1, + "train_csv":"examples/asr/hkust/data/train.csv", + "dev_csv":"examples/asr/hkust/data/dev.csv", + "test_csv":"examples/asr/hkust/data/dev.csv" +} \ No newline at end of file diff --git a/examples/asr/librispeech/data/librispeech_unigram5000.model b/examples/asr/librispeech/data/librispeech_unigram5000.model new file mode 100644 index 0000000000000000000000000000000000000000..adb260dc28452103d05a48309e44bef0b22ffce6 GIT binary patch literal 325118 zcmYhEdBD`u_s7Xnl0qFzL&K=k2f68&k>?f=6+o)Z~ zP90b4e^N}X)^VMVtM}jK-{hJLC!AYp%*di+_n+CZL&sGkng9HOGl{1k5{0z%u4FuV{25UC>MQu;>?a~cU)bgN_F%ZwNJT%Z1}|J+NYcxf5oM>PZZVRf^joD zcJA0dl$@wYy9dY5?6_9PcA2!C_T`nevRf089J~b5%Fb0}oqw(hEN(0L?$_6%HRd82 zwgj?c$JKI$203ug8*881idFe*<)m6!t+`60T%5k8_BmTD%brspsdYsjxboK8hsH!& zZW?i0tz`H(b(7ouW2P+A_nKV$n9RtY`%S4)@UrG$D3e~YT9NJMJ6R}JG7J|b3v}H7$*;@Ung1#vSC)r{Jy!!-i`9Hu zPI?3&?xk7Tufy~8>}bm~K6V-go8^_ype_5>#-hw^d7+j#U6kdGvuYoU^q%`>LYnwn zb2RAh$GljJTrJ6S|3J}9&2o3Amum5n8LE((25n_DQ$yKdcI`7P6r|Jd(AI9LP=oS1 zuZNtSwU#{R^O=hVt56k8>K-CPrPBxgNE&{DQ!%5Ad# zGN&1mW%;~eUMMXk>-|8;Vn#)JOw-11%?}7ju~UVe5V$@NCoA@520y?yzIW=yS0RgLYv&3hlCrZ!=DZ!X!U5# zz}==dJ6$X=uwS{)Q%0GLJlXrbTJ(luF)e=|1Z^!=)NU@Hp9o+qR$7a)+i*xqO628= zOW&_0ZPJFhVlp(`CFR?v2w6XED9YB$JYKrkT9CWDeNdBqGQ-@Mk@7GQQYK{n5<*s* zHeko~KXj7*-@X;nS}ke}{pp5tL9DjTiF8J0UI=aMC$bgTf3fRFHK3uKVU%p21&BqS zXp@x(L&h0Yql#>{*2gtEr5)wR4WKQ0BXwNb!#_t==AnUEr&9@{Z+LOeExzOE6rNg>*JxV_RYCSE0_EN#4@Mu zU+?;9$cxqHmeUCu*UeP2%hixp$$XKe>+uslt9@?ebH8gKt>kR3j1cxNe_o59Or)A* z!L!hDtI&XZ%!RaK6V0;00;pKz4GDT+`%M#V_ri2{_L)Q9dp&|p0fL^a- zM@AwGO_a%JWx!~U5+>TW7em)1lPfeyYxG4}@(Y&_-< zQpr^!={X0I0vRKFEvm_RZmJ^_@Zp}Gm;Iru#JyFpLtv?&e z%BU2Jvg%SuO6SXOT^H9fN@ydV)!*rSE}N0%6QGF-&kszsU@j$xuklr_#;T?06t#bQ zAXBZ1GNp@uuQZlbBr-e;6mwi8x6=kXUC|o6a~ZU;n1-+Sra)SSw1>CJk`Dl21f8hI z24B~{MUARA^ej05GxYiM4+n_bsx1@AanC`MI8l}!%RO#X(KRa5=bKvGd;-RbS3_IL zRE*4adc`9&W>B@hdI#E?8x^ThmALy?|F)K<5X9fo4FG8*lB@~CM?%E?KAV*ZFF|Tg ztF*~xANZ$yrXq(gfUGqu6Y_fBLmMy2j23idkMC+gwpgWO9ki7Ls5BJzdmg|FixRDJ z<_AtTP{2hiAyp6s`MK}+wRxbFDV9jORSLvP&n8kCdGK&(E1h+a(yzG`fYkY%oH^aU z!3LAa%kw>Pu|VSJEvGBGfK2K1LoGA91WU$*UIhh)V zn&gd3z^ry{ROjFckT6uv$+Sm3b)-}1M>8#Ti#+!-A)D~BiLe+A-sHy`Vgptt9SLG> zn$Sk2Lff1GWVJ7cHeJ5Z0s29ieZbEE#!QJ3u=hzzYWy?En&hlAoNmZ5(9gfg=`d-3 z@PN~;#d4Ee+u^5BnOt6u&Jr|HFbA(M;t&MGR%yE#hV67(&zQ~b8o=G;6 zYcB_(Yd6ZR6FqU3+3DCDAY(2Qx;B3^2f*4el8fd;P|G}<*EN0%LB`EH+Yz)h+Aq_x zECtZDV<*-^)l!Z3e~N$<*(x&+AY=@px;4`bTOI^LzGw#rKT6OBLNb#|==^ue2S8T8 z42{3x&yZBTLc)E!{9cn-rHD*7+#1@V>a;s_AOD;#qED>$UufgBp{SkkwlN;Rf!${E zHpq~5S#DnGaa)VJDfp}7(pugPXqef~ki^X}Wk0qVVfZiRWzEfFm`<@%o5Ab@pFYHb zD1)nAWWh>CzIufq!p$o$$yXub*-tz9xNn?p(vfyp*JYs)c79v#NYJ82MHZ`{cX2wC zK#1Gy>2#DsrFyd!+BBVFqfD6$1-Epa5uKLTq@7Tuhr0C30$JV#oyYR`IgPp&$%u!b zO@{xi=BPy1SpX*U6ujK>KBSe^oRBmA_SCHjL~?oeKWeGl=qr(Q{ttkq&Qi%+_Jg#( zk<3)I1781sAjl{TvwIGQh(|~Q73+(U(Ar=W-0>R!#sZ#^)$epVk!O>B)CPZsYSW_* z-JGD+A(?5QZJ%lYuwvQxYilgK%#lnXC=h$y0Agk56X`aY{JCeszLe!_>Ng%C$-t8b zI{g(m4;s*qT_EBn(;eq+n?lnF%x;ftMvzM6>CkuVOxVN`F_%m^0@4^|bw$|SPXI7R z*@>2^%jBy(N=q@vaJ>EAdK8osGTrbY5Xz!GK5PGX2mEQ|9OOeE9p==d$iMO8d7e$*Xf2_IWHq$fYz?9y6qb8LD-;XYg`EOecvG( zx>ZHe`Z18zy+8>|K8LirCz@5=AFvDnc9@sslpZTWIL9cT)Q7N5Q6;&cKLm9}&w1(q z3luCfnuc%oywvFzxZ>T^UO(#gU=vwmfON6cSHY8+~51>pQv~aUbU*U8%ldnkY zS}SY0s*)YayIlZGq|I$%5Aa$V#Ik5)6_HW{v=xvUBzH7HnOxYFg+;l~fpv&ARre8{ zMgnP9Nyy3L2*&kbjZe$t_X1e{tg2~wWNJM*dQq8Hz5ih#E0TFze)b&Di%IeHy8qO8 zPUM?aGtO=UV)9H$(|;i15_76oEjSziZji^X$6Amr|J`R1w6?3zl@~N*aT{9+`?>!>x^T=gTmP@kOCja>pl9tIujdI+ot2`Hrs+1`=CQ6YE=mcb} zv7#dW+t&wxwQ^NbT?xkHh~4rPdHa$o%H<`3Ci8Npl+Pide=0y!xS}0%&o&5S77e#7 zPabYbnF_xfMsWR(YqPFlt*~Wn?-Mkl#`T$ZuY3DvU0ofVSt#yM5-b`&ULymlP^jCx126y234fz z+t4P891`#NKBU!ObtkRz^pAkFgJtC0&j?x%&Qcovqj4dK6_+kD+HO(WvdgnyoAI@! zrHQ(JPtbBshCNqB?pW!G*^+1E-E~&2W!99*lw^zbplM;XA?&p|;kd~%2^qa3q~*p; zSE!(R92gZpye|N$Kxs|A6*J%+$WgR|Go+`ZltR25GO## zY}g)P%iBRH2h+n}8nnob#S$C7yX;d#gABXdKjn2F|5cqPWe~YSgvE%oF)?$j%v%dw=fR4|4EKGIH(yIz>)o1x) zu|EAUkB~}`%>(B^BfO0El40jTSnCxMES4FWeLhIs*E38aNc64CfULGmGrh+`nJfyr zK5jJ8zp{aNaT1jE7M-GBzSTcACZz8@P{vzS$|RcP*Qo$;XVSc`e#Gfi0u^A^W1c`w z83lQ0I)L$u(O1t=HhbBlB!cbi+qnSUIt9CK8%s=0;HVv6Z}gcZamHDvhkY2W;n` zX#p9Zu7569GcrEobkHddszSqBa}HZE!Fb%O$wUWAmtppq;AbZiGXPJ z>4ZEx3d)8_t9*N|$1SlUmu2mX0IZJ547QfEoPRM01!K}5J&vFioYE0cM)p0f0Moun zzg<4w6Q_%*3QEnlSAv)f(rgR5UgZ(cfeL6NlWqesskG|4w@B%@Glcc}^t&8DDWUZz zKIlL`qiWkvj{?+|sYs8882(R!)r8G4-;AfAV?J2q)_M+-Iwewa)C&aTc{fw8s3kQ2 zI*`eW!P@Yl#bsW2`cr}?x&}G_cLRRAjMiaa^dlb6atyZYz`kQz4`X}k$ir*ky$%FAZjDGhd$s(JlU6(o?rjlP)K&w#FvgvAT z)DlM-c9dy(e;p8{pdI_N*ba4Sg0fNb-ztlenynYLfLTSESIY9h0Z!|dNp?CO+RCp4OKbB802@3F8H(q4 z>5Q{I4l^n%&B+%4SX(5i(11%Ib-rLcy?X^=ZHb)BnMja&GE=5y)f*v9o+vSSj=$#J z3=%hJZjc;vmq);AqPC-To&Yh{Ns^W!FM5JtQ<>ulh^|WD3qOI_f)EVerrzH82dLGM zlR$<~?=EZBYDl!Om0PHg^^s&IQLW_EDAle9V6BT{PFtb0wL^1NcIydc*)nG_sl3z+ z!0MC7YP@J0NRwkCul%=(>GH~kNac24qT(FhCE(Pa5WgpJi!y4;v?kj7Z6t~Q%q<#bk+y610zHp#R$nu>eu zWFQl!9$+$GPxUydL@7++OP>d_QZj95H*+EBXax-E^9Y)3*jTB8{M>vH%MZ&Jqw1Nr z0cyfxO8fI&Xv+*Usto;d}_LhUFa56C+l& zhnhgFn(1OSS&@&6o{8F~Ol@f!8*L%!Je2Yup+x00&N|N z&U)ccC^{#Te1nt^_fJ@jjvnR-n=++}oPPv#EDH*H`Y4Z_tV*wAJaR=_`bMR#4pCk= zR4|^^rXJ1%WFMr9u zB$>Nzn**s+BzwSH77&gHD(0-8p^$4(KHpgiLAm((th0=u)hiif-s2Adl?x+zPP-1Z zEEoz8tqx^TbsmXik2L^{tukw6VI4?P>OgqEGhyqP>4cuM(y<=j#M85q(biY?0Q>#Ss0@eGT}~WVf7V z6Kb*=F~?yr-1JGXc(hlO12c4wy`T471{NC&GDvAFR4t(u`TIo&%IGd-*>5g@ae+nX zj5nQ1=OpnlRNT)K&Dtwo{Tjr$#D}FIy?%#|S5&HdBo&-B|T2fm(T}kAdq-}d>L+xGr-a%<}YwYO}8D z!(Z{}*7|*C1H^5q?Y7fv(9{H9!kHSh(o>DqGG{ge-T)!*6n*95w=B#?SN?vNu(4g? zbnp7lAZsI|M3>0Oq{Tp%SB7rHSedv4Bp!b`S+WvJi%m$OOGmF_iGx!~cXI&0BBr>_ zx&v7LY|4|`i8tQFA=iR1NaUkaAmZjKHe*GQ zDI-Bts1$HELE}JO?l{Ok7rgoAy7eK>-n%nOgL)daE+Qc?=oF|V|rIzcjwCFje zoLeUnw$|q$Rfb;gr0y4bPjV7R6zMz#((=YcQ(^77|1J=?#8Yy?y#$S^tZD=4hn;TK zA-elB(AMy*B=o|xjGhG)4+u_ZD)RJf0Bd$cjUGg#17G!gvbxpWX|B_129PV~IgO@N zkO}jltw!o2Fz_u%5CV>S6ttJhuhzOE(?bLBG_>uC*hGmC#s~`AV61=&w~B z&ZqYE2$ibryC;+}SW*Tq-w)bmcQ&z1y;XR4aR8u|8>JeHoEJZus3VCs?f4I8L98U~ zV8P_SXCn|DNNtgumOOzCGg<2>M^a3J^ou@<7%vUFBHVopbUYtaW%Nla2RYJe!$^M>&oEZCYkr;Q1s{HitfR6_D}5 z;OsO3%DR|2t{rlxf6nIQ!^fa%?G)^^Pdw*{9w;NzKj(wcgNiM(;$wo=x74$0gD)Tv zn_5Kre;dbYm9cS<5p%%zps`?5GGGamm4+jO4#n}mfS9OC)pD3^Co2@o0b?~MFy!9t z4?q)mS^c^fweJ*4R-?gNLfEQ^fshe#!R7##X{OMSS1aG*9f7Q5Z9+X}>jPprw&XY% zf4RTYQDd$ocMb3qrZq}K+G8+)^U|7yHen3pFzwJrC)pIri~~G2(_KMcRN9y-#9OBO z4+JvCv6;iuMh7`iEwB%gqlN&)%1N+aQM>;C2WG`09$E5lk|WdMe5Fa$)$g^FJ-bw6 zE|ZmYPjx^|b!qu_1VG$Z60-jJ{<$R=;U)B+3jnMrj+N!`8u$8f_Y`C7px>$4{w1sj_~>VeX4&&vCYf!Y0xIb zEGnt)JCAz^hzjQj-}5M8E3Oj8^?6Tv`hwayZ+P9)qYDPZ?dER*(YvcDx%MN1R^1Zf z`{!dw%MCjMo%N>$0M>~zgHpPe?DVB2Zs9oXTY@ob#d5GJrGM}YGGPwP{Ov#jt4CVO z?bfN~SYpcK(Ca)!Oo}0mmV2RMCL>nTvK+WNv<}FMY|-9=xK@4BfuPP|Q)pf=frY&MV}RbW1p>zWDI3a6-hN)`>b zfXRahS|4es%7aQ-hH68Mq|^prOsRK&Fh_lS1ejHhp0BIGVnrhLFOJbgOS$&82_cJ zOq)`t60+`HP{w>Ylgflq`iEjB!4xHYb1$Uvkj_MU>bv%I&thOgo_-d}CQxjo+*wd1 zf5RDoXj@@|yzwOnt9xGW!6e^;HV)OVuuU%e6d>*kRFdBD(q|yXS24w*m|Ab&R?Oc7TY-1AG1* zcZIb48mej&F74*u@newQ`#72A9Dne>9z7jA-EL0-(05grIx0;#W<@tmcVz9Gx=BB; zVXJR8*PxSRt^veKHUac*1TfBZ{Zjp*Zw1J5#JbQX=U1WQF2T&aMJpt7!M1n7fds8l zvPC&@2$YqU$d zOu0p!p(-~4S#bk1Qkvuun9ZvdboI{^vhq~zlLl@2{B*)r7%M81%UMqY#L`H}bxLVnXrrs1B^*nT zs$H$5rL@iJtqNGmQcj+p31wr7sh>fkUX`oA;901vz#1<(sTWArc@@$)M=fcU|5KX6 zRkM9V!*oQ>G}foY{SOzfFC@osK~-{$r5O5^-?YsPUqiM z2!_JtgGj#n36kmr#c9kh5IRO%-C< z^bG$o>i}BC)5Qk)sWT*5b24;JH-g4^um$Lp-ghG~V>EaXpfF6`1SszRVM8U4))Bm4 zb!RfBCvj*P^bz?h4yzS#TyIYs>FHsET(S#De1tOy8wLJ541pVWxAX{4zWxtkW0)i| ze!n_Nm-s`;c>Gj~@RODf1A&aeGMQo`NymD!VIfr8L?7h0hc*R*I!uL4_TlxC^OM>(Wz+V*QdM9Ll(3q^D6& zRz3CF>1>aqhX#>+b&dmS$7h>%F@P1F!ZndWy5A*^8m+cyGu4HH3m z+V>7H6D>u{I#WCiH|Urm$J`BI1+gG#bG?7Br(ofgjqmda>0CIweD48A!ogabZ2cIB z)tT)#jXUN^j}lgqicFdZVtlZ?%Zm3QP4H2xy#5hX+*B1-jaeW2*VcqwyTGY|MVY0P zF|4&#Yr$VesHF|3!DyecXbXPDFc2rHu z)4O>TygRS}Q~!PU@O)Z>Kf#uJ0oY8U$8{*3?AX`Y8?YuvJe+>tr%vmo#6S9bF40E{6#{6Zhx{(nz;35 z>9i3*CSNSltjUV+4xH^trbi@pg5>a88np{UpiE!a*$cw~5>bRxI zBQJRdrHt;kK6>3VKq;@XuQ_I}BYNGHP08JFSq9iV##xu{u=|CX@TyFpF(y}ySxsSK>jfPXAz8AqzqS6i>v zEwJyUnyBxN?SYJ;EFxae5z-hU2K$~Fod8UTu%mbWPFT;Q=tj082W;Q~N7k(YIvzMm+uZX>GWJlPcI*XWY^xKBbl=9aDdo^@X*$e;Sm`!Bj-c^UqPtVdWJO#P(M_FB z-xnWmNfK4L=yZbC8qC0K$1XV+z&L58zesdGwB?ED)v59R%R%6+kz?w;#}YPy~ROQiO>wo zl(gGDVAcys@%Vk70(~E|jMA22KGPr_5Y!u;oJI8?1mu zVZLl(mc3~vNX#YU;F1?TgTKCZW~@^A^JR^IagT8|Pak>9GKs z#>{2q(9a1n=LUbC^%p@{{SeUBi=Dz-PUEhK+46g6W4)No>JDtu z5>L;%HuM)L9o?`vTf>yk^j!YUBI$}0$@t%)OlBy$=uSPCIWVxQ%jNPv90$YK*^a~dBQMp(5UTu0a-~Z>y3AEvXKe=M{gQ3};Z5(5C@nWmUx z8f=pzn;l|^g!^JQ6+BrhcC?IKPy(=K<%m%Qb4aVF(nDqy?tnIjG)k4?9ytKW_@QfY zZTxU(lM4EU9P~d(%a()x2C6yuSWi&WVqQJfW0<8-jyla_l&hrJ@+_yZ`&4AqIZj87 zob&#A9<+&|P}I%ss7nEihX(vR9#XOOeW)lPCIE}L8f zY10nwTdW7_{I=|Nz_?z)e@rf%0%E1G{UsNrlom}!m(y0+ZYDt7c@dQ^;Zy36dQoW`eg8bMjNS0CL_x0)e)9&PF_+U_ z<}q(Vn}m|&y#3q$In%=Bl{Te~IaFmte$Kl9FwPSzBj2-Ng0ZGSV+cjC*ud;DQz0uT zY*;`12-*r#e~N29_RlCWd3oXsX!6C_z5gPDwV{b=oVDmSg-po7u{M$gi$NGwdUk*1 z*Z!4_k?QKdD`JJ|yfET7$e6uqAqZ#2N+4qoZ|=OD{13DVD|CerX@@ns)@t9FP)qe? z>jPN6yxk!k1Ty9{ukLRV*F%AfQ_d++B;==_o+bse=WgL?F!na;TDf#9AnPFJZcm$c z?&Ya4U&*aoJIOL3?Y8wy^w?0hS`TmM(0?%+YM)rPy+dqjg15l*oq-TDist>h611kR zMpag~&35zXs1V`MV)Z^iCZ$LXNez2Ajo}8>wJ$UiE|DYoW`KVTZuwJE&{hDR-0V;P zuK~bhgU7LU(VGW?)M{Q#1{>6t1&@`%vVjM}MpmBvVlkAuX$ zL-&!-KMAdKOG&=apw*7ccyjx59y48KeLU}ZXv>{D8cZcJY!-+yfGY+4YS&i)Y!PnE z$>y`6tOLics%3fZbufxU1^V9{g2qXc&N4qK9Y^PkncWQ6Ti*b)zJi%542wVKTMn2R z&UxRz4&($$ue4-81TrSsVXKer>;)ikw)hdGbYHs3r*%wk7U^*7__<^B@N%$)CKiDi ziw)}GdBeBR@f1gAJMufoxc)iy-<|&hfU(OR3a*J~Z0xugFS(-!q{*4Rp5EL% zpeIPoZ@86sy8@PBXv;IV@CXfDEMOVCEy>K1mN$kvfSILSlEH@o zSbmK~Zt<7px+6S_B4s({7y!+$D*cZo7|RIzepyFMqhh+4>2!VL@euJufd(=CM95fV zI*=b64sD`PtyxcTmMUcIhh41fb2@-=q=TPUXi>zXuwz0fOGbDUlpH{xtJ$w{r)dbdEv!oV#qlD!l# zVMJBTXS|JZD~K^ZPzTq#w|f#=6t9lbWRH+8b5Uf$UC^+^23Jb)k>LZWRoJ*%^lr=x~DC+5h%bfyoS*QlV3jD)o9*J!t8^0^|wN;2-;XJr4V2_&elF;%G-L$OVfC+H4W~L#uu1h{d2*CA>ORqdKXM z1j|L6?ERz1p-bo>+~y|`D>~PlU{8ii!BQZrcNhjTe+5817&*t0*{et`jjlgcRK_<} z1+pk@sx~ZI1KOAg*VfXq-?||2EKFk$Sr5_*;r0Uct8VDuaSxInJ^XuS5HgV=TLZ)` zR?1}1RhqVQguQ}Rc>Ina$S9YT4|cI2hb@=yM$n34f5%u`XCDB1j4nER_xF$bWC2$S z_5(11FgarYPRrz!$H62_m)j!mKt4>RE4kSdC*mein7lbXt@Q+!d91iBlKrZlE>lp) zv4a(e3qzOEo$4-ap01cgnbp0wL zse&O-9ytZT`c2rN+J%TYBS7QU)jh}GBcYj=gGb&@XL}wQwmEsppX0H^Ub*Q)kH8?t zc>3$;e?_1^Ut9!jD;7G$-D9Dw!bt{$-VUEN9!MLlD1`~07FpzFw`-xTn(DfJ@AZ(@ zkQK}k8F}F*kFN`yqqFRhir26%gYv8P4;sRRCcIf3=^KhzAAy zGkJh=>g@yKx098OQ0kV#&vfSf$dc z<*oiUG!o`+-fQm=v>cOFyhv3`JMn${oXkn~a|<@*WZ)u#CJB^I+2tF^xMpEK|FsU_ z5r#Bg`IZ<~YB+XV`W-Z*OxK>3ztl6M4m<(nLY1G&Gj@kp-IL9GD*Utred7^NHfP9uKWbz>j5BZ(Z z&jHkD7SJC3JyEJ^`{XzG1=_IVx^(eWuAHhi+u3R)8|}{rs~;m@b{^m)y+&TiKpKMs z6D%6H<()=VC@QIG@6tm7taZ)wQ(?-#;Y2{IDP2WI z4~L9pZ6>8VPXe*Zl1q~mPxAz6M&O}mLK^oh4|s-DWc(-~dZy}Tr=3eU?zU=VklCX> ze(+kR>E2Mt_zKHi&}?2G>+v#!IHrfyd&hfTxGf;(SFZ#xX4xLTR!xde?iPYEqWY!`Kz8T zQctY3TszkhY%2_KIp{6VDUHIcLw=e<#*pe@2wO>h+O; zMnz2c+(D3D%OH(QPS(&i|Af}&#u;d`hHV9Bjs3IThPB3LE}SX z*hY-+6|gLFQ9-ue&&ecn*7X@k8yK}CY}mLj=b2!lCX@C90b*h6C9`!aby}ax?bHHo z!qTNH5nQzPY6Y}zYqvxrNe>1xW|ABVT%=^2t$Nwqa1gY$8)v8Y4TWOpv9sbr(vc7* zTr<9oJjK&S%EFbWJFSz5{HCXc0JGwvN|jE3NgY5XWImFsi4o6P%!37vgL_@$P;7}Y$nwD$%~hOzulJ0ZbTRPm za4Udu6K+V2y2CTUB&28b^1mrSR#iL{Rg=3(5mn35vfe!et=8;0(&7F2_3rgN=uBD7 zChTZn}L_6>sb_gpExG6!vA7n}cS6Q43W6S2buQ>-4zoEK8}@i_Efd}Zv0R(EZ=e(K8uNi- z_H{RN>s!#qGG}@Em{eL;yzLo99Mm!*Pk0xkrfkHex5>o!J*^!fAiM=1fLUMAIlmz5 ze&_(ZOJ?v57C6B65%u8GFCEa+0SuCh0pd1OH-$+0egi@S=q0{2zb72GAC|B?mOz?d zI3dK`qG~|*-vF(asvJp^(#BPa2c7crp#oNfT?CMy6|rp7bm5N6{uM!8TBM&MwXskw zYv)i$cd;S`^p!mkQgKAK1yWxe(esr62wA3Fn?#A~-) zX_+0nf*RYpc+$PbD`GX}mKaY@i)W=5vNfwQ8vt3&c*s*;-UQN^#^_9w_1zS}GF2NY z;_JDYBMfN0yjtAWkw`D^7G&y9AoOeQ1y9$Y@t`A7e%u*4o}a^z(kF!a><(zf^0yW= zdzU={j4izudFft|))6rg#78NL*$iDw${PJV2UURBU+(Py(+s=QXB03-a4E`5r~c4Z z1hwb{&@Ko8T1+#ks%qq{^N&k`8av=0ItirNM-cw+{YN%?XcPx7fy>yBwHmicL zbmVl;_5EN7N(T6qFkV zXnB{b%pi{+1#L3Tq90&eJsQAj!<|RHhIPXKKrCBb!Ph3(MeSgT~Kvg{J5TH_=0mT}MsLk|aKloE6jjCvPq&{!i^ zoV8`}c#v2K>~eFQ&z+_a@tLOAzRJJiI+0U5Q+fiBaiT9?m1Jri2$mpuej+if5Iwts z`>SsPG4>kt3CYB5PV=yFBtve8wm!kRLKQA}k?anS)1oG)Zd0L6Eb4jN78Jy9rUAz7 zMV#pmcpPfZgU)KP?fZy>Y-TgKXE32)7y=9T$S+s~ zVeD|?FE1*o)khm1_#NT6v&B9`XD)FlcxBQyPb*|)M(VYYkvWQ39Z-o-W!C!Hqv)%; zOu?@#cO;yMr)A!so(O+@>N0$V18P>{zok|JnCy9%AIG3oHm()Jq=L@-;c5UDp-msi zudEJjRZB9CXDV4^E7zQqFFJVqwED#?Tg&MzHfZ_J+MYm7e{6Brb0EuPfjOkPhG!FI@9qg0=iCf?M*&NK`6XzF8|`5U(0$7C&t4GB8+hjK+piwAS`Po^ z@ZkWUcvj$#hp5(fikSGq8NH;F0Jhw!8ypjqYM}E6dO{reIYE^Jnn37MG_=gj6Sium ztJG?rGPHHiU{BU+{WJ(H?sjU;rMpZ#0LbLba*V`Og+Ka0K&r2Y_Hx?65SB3yl11{c z(s8eB!>-#bk1Ar>23JyYICU71Wy>laj`GuT+>xGBL3dIoDs4HjHC9__pQAxc5){qy zH{%!p8;}KFVL8?lsAoUy?|r-{&`oKR3{b#&CQ6a?J^>O5v0Ta)Cla=b1n-G?!=cS( z9m5Njg=#@xDVlf&u+@gnz%mudlCwNzQitJ#XG7QeJ6FtPjsli-O6@V9jDfbAm5SUJ z#^5;?#Q17LgGpTB30n2G$mFY_t+kXF)@F{G6|*GzlG~7pkj58QpuD>N^t#z0wK8fG zKXD6?v4;AYfs1(poARq4Ycu` z4EI?6`Vqh?uG`WFmq1!A6ZRb2N56Ou?9STcxZk0zXf_>O1c_whKY%P#%;5}*+zO9_ zMx9b^Z|O=P%h*o+<-k?D*N||ffC~ewfmkh3yGqi!Iy7^Lp6)kD*EOM-oOxy8Vhz>` z)Dy{f+XGncJmsU#lS4azSYCR$K*p})G4$`07%t-!AwQ1bJFn}J%S9%yX-Zoi)xjvu z-o1BMk3usx%B4!1+-O4Sy`g^&4-TE)!)YuPX&I@sPR#nS<*i#l7`H9lN@G39Zv|qx zp&rNyN?QHY--hk=wg9nY^qr@*dwUMy4@T0`eH7k6 z+B|wm-$wsT=~ztWR<#cI1T63L_Ong~ zu=??b9h{eP5$6<#^uN}0p}0^Xs|5dQrmcVJsUY;};G$7F%~Np8He50IXoN$}Xs>Ea z{3jwWidRRdZJm*qJ^l43so2rQA#N9J`;w%N?nO#?L zb({b5GLM9DfxXJis{rD8NS(jlyaw8Y&0$lx=W@dYkHVjJW#uNNO?5)<&nQJ#iE!k* zo?t8lJyOw)%a4=6EWc2ha*LdJr$;lFAELc^FIYUmu)DtTKFGMfY*$%u_k04xq{qve z?E1nj)1wr%I&-OynSHBg0OGO0YE~^Ld2QPvrrt2!9IX&MbC~~$24kMhloiRG7r~5~ zWVKCpf5kIsQribR-uveQ8GoqAdc&ymn;;Am)>Ij!Vf)JDUq*+nyzK~zJ^!32XDVWx zN4Y_aoSi>(fb)CIgQGtIFwTM>rC!#b@~LBX_mi3ON5!nl!8{3c?w3Hufc`fWW`+Y6 zc_iJptEk2)6!(`_?HhO2LOr{F55k}d*EHAr(LZO=d6-QH{{&*p z;K-{kOY)#XCQY;p`Rf-*>pg8n>F~Sfz+V&VQu)YI|IRxh@BRs8`7u@UELdK8{|yxL zjcSpXbyr#nIy6Ig*LIuKQiO-P;dztQfZ|E2$b5InnjR&%;3Kh>9Xv{sQ7%2#f`&^b zAolBpb)alaa%{zb)E+rTL2I92+|tSJ$n`y&Fm=7Wq0`)$Ym%Eb@igq~an@$7{ChL7 zTKk!ybjG%xtgPGS_Psrs8GjjfYitijL$EO%D6^EX8m3h1-)RTvxPQWz&KEm@#JXQN zVpR8nIlFi?++MT--zcKXG>`7xv@79waA8v_$cGBl25GCTvKu5EB(PFS-@XvWL8T3^ zxm*85>*cDsbvjwZr{~eVh;Ox4Gmb8*hwTGoJp<~8{h+Kc6ad*{e@K&d{9GDdzIBZ4 zZIsK<0e4P#Jnply5vBd_PJ+dAoKB3*DQGL3KlKZC=!FVd_O`N-?DVu_q3`JcoSO$j zJ{&B{UIoG?AO1Fmo_>27fC{jZ(5~-PpsWq_SsSYKN{c687eRux4|afC)ofulSHRe) zc{|8eL&0M1asH8=4zJVd9X0$&&!p0-E8&_)1H{!sh0*on#Z!Pxm~1(u-Km}hT2WCJ z*Xf`>Ogw|{#=0(5us&H4Ya1-#Yo7&WZ0d!B;F5gVxnL%Z4EE!)yfE4`p<&pj$Vp>B zRI5nHI~Nl~jtqI(xS$bHSV0LC3OE3j-m3B+og z$m-LJk@UL-D9%*ZwbO6)T+-o+)vHrHLX-aK+R^uV9<;W;vMTo|6c79G>ZfX<6YukM z99`leFwN8P4hJ20Vp1ItzOJ1KV)fD2%hae}k)NLjw1t(s zkvbzOcfTrX64k%<)T!aZ7wY-xF$AyQyy*GyCOvnJR0G^aL2E8v&7&V2Hy6OHAu8Am zFRAH1_-BPJo~|8bx&3W`T0iFw!rw|;^>pLE-8+!RkvjS5qd2K|9m4f0Os4-;D4r(t z(P6c7|Mel5m6XplV*=djBL{eM6EEqNp8}W+U@@!5YjC*j=b$D)TWn?9MIa_XJukNx zYW7(SXndE|G*JBx+KN+aEvIGQ1H|Kz`(S!tuv8(-8l^JR%q7PkJd$q1^+;`7g-igP zV9A3&LR!A+qRTt=zxsC;8`)wxq%o(>sFkS5K7V*3{WJ2koTWf449Rs1{s10bYTkKP(n{`jz)WuWlpKfvUYh4HU z+hZDL?ezhy#8%tpyrzid$-CHXvQ+6>)9GQul8x#SbQ^X|4``+>)d6qY!oN``+%R1zHeZo=Mv1C2=zCG?3Q!*V{z8V;K&(p%S#XS`R9~@ zM5L~25AKyGT$?$9(Tc8=C;-~1(U(WrE2!aN>!L@E?*uZ3k8c6ATC3ipt|dzq(#BQ4 z#-XhcCa0*G$I%;Q{NOsmT~ZQUr3n0JJA6OXzcTHphP~#IAhBF^XT|vKP=}aNaqh3k z#tOwlilf#elr(V_dCX5HD{XAx+oEG^t)oCpDq(JvcmD@qlCU?lbk55h2WU+n;pV_N zN*iW*b3)xFxf6&Jzhe>}o>W^~gY0}Fu<>0j=s`Ju2y-%!F`3~1Brh4^nR9BLYtZR= zwqnL81`2&Rbgm+nJ@bb@9lY0>z%je(uNqD}9y|xoD#ck3XZ7oy>i}*(=#LvHU@Y(q zsq|FRilgz{X!{*5@Dx~d7%8t`0zh*zy`|;GvHn#LI=G`c9v~KmDzx`q=@Imf2k&BC z1z=(biv-<$*wsK*XqaWva>)co^ja)z%(&JO^;$*G-2ZydNcHOs`oaM>0Kr;PcgfpL zf-p{KfU?ZJ#c7<$o8)7qttsg=Txy_{PbbG2RLc!AV~R(h*V`<+STPe9^_AXtLt16{ zBaEO7uB=1(950#EY4*L3SXRGuCiuDUuZWe6FD2uI)pI}{3;&j-8?_k^c*-czqTTe` zhrLAIO{#7`u@2F5)r~@yKS?ZWcYLE64jrFzz&2l-&T>HinU=pzQ^1%+x~iogIoqQo z(z=>{{%So6r`5=KlQ|$(f3B}gEDlr_EwiMxrtC&3>s{oo1N!3X2|&?P`|Z_^hk=O`VE zfpZ~L=@p7tM(9smyulc__D`NsiW7Ei;K{#wG`b~=F6v>xOdJvK>#J%RM>KITvM})6 zsVU?7vZNugyX%O)9*VEgygI@=Z+fKuxgt!qd5rs`{~&DTg#VfeuaI>A6Oi@Yp7uUR zskmoxW1q+Q6tGfDXm9kJtLstHm~nzT8-ldk<}q> z)zVFXJrcsKdWGUva9g-1yH0xtbZf#xP6}8?MZ9G59U!em^xv4}jLy(hl(PaEt6}>V z_A7(CL0ct)DIt=ZHvqA|!QZKeTk>PO1F@;$AK%{I#N%hudaFiW-=dD_@WI<}TM)`X zN_ky6diI8n+dqY+OZTh&6|+h)lv(bI3Lxxsut+Z6&azLvM$c7%itH|nmc_jT8(O?=-s1}LkF)e-jfgrmA z&&%w(uYWD+|3;)`SRKK{t;pB}apGRWfmB&K=l+$btnWj0Ejps^kt{{M6tUXzLWKNU zhO{=~J;+GzkDw6^K1V2lOO%2my%t}s5H`t{*>nU~-~laQ#sOLnCtcqS2C%-3iWUDE zU*bT(xJa&5t#J^fW=ez3*RW-(_cp?3WFP6Wwk() z;3A-UQPXw6G56-5LA#7oFix#YFlYYPgBZ{HBA#5WbUZCFRCTmYQ79HnSV4Ba(E*NC zSSfnm?1}Ubz4Q=qok?{_O)Cu1<8E^(YzyG$!`mHVpTHzz%CJYp(S+yt2*PP|6QI<*k9`v;{EP*EHk1Upm_Jxbxd%r^+Cm@+Bp^bTRmiNmN! zl1f`G!xbP^V-uf(S!QY!p#0T82Z<+M6qV~1dIZkw^l7~>|BdML60%4U%Q$R7bxmKa zm^EVDDCFJkOOMDA8a@ro3(tH5X7%Ga8;%kV{t+M^NWn_pCci0ElY#y_Tkcl?W1GVO z9Z6>_12G=b`qaAinb-dSGaht9lL|xP?>_-+<5rjIzns=-S&sZ0y4EDYJxfks31p?A zNU2R>WgWrnr_+6he>@v~Ls4#1+S2I%tD`{lTy3)&5~{_6(yN0Q&*3JH?sIyq?HCGX zk#|_qvRx;zxcU`r!Low_vA}f^*On}I28-7MeAV~w;%U?v+a}Mi>j19r@Uvl8fLMn5 z*cs(0V&yb&qMDVn>VTRy`0IHgMr~*&)kKEqSL+UF5({b}`-UzGSx%~Qah*p2lX|O8 z5c8B~)zxP`GLqQ>%2-h^KB_!mOOLO{3ORclXe*c|;JsBHqF-+dXa(zCI(@bN2ZgL) zb>hJyw7mmZpeVWT&Hz>xV^m*p{8SOkCcfq^>+J$$?Bbf#BFQ7M+|*DzWo6M z-Yt7NWD^fB~!zx%5qXe5xC?#P8EU zHnC}!rcuh_A6b{@z^ztXN>zurj|PI6#O(1@T@P9c9u1c!Znwx9MG)&Y12OpCp=3Nd z^zp);k$=we?(NxC$awzLC6hHor`x{=c`P>Dl5g`&xaJYbnuDF@QHmzH^+2c5Q}fbv zFtiDQE`{MTceDfQRmn-hu?|o-mc%<0i0ey3GvRD~TpbA>3Q{-$#Ok6~!sUb$Jrn%J z7y-MV;t^PUa&ofLZiYu82;KW$MXh`AXFs78Ha{OIF5R95SaAV}RiTlUp&;Eydla+F z%b{aH;x4S_A{1-X@rwX0^K9(2Ipz}2D$5}PUHDSZN-z0wzIwxD9z`pkmu<#*Har6y zoW}1N4`j91zwVUnu7R#ika#Vssx$98Po}$2-S_mk0f_#`)=4_tNZ2?ISCnyFY?I}a zJfCr zhWg9^MaC$m(oe&2&h$7PO?xY3xl~kdmzlEwtSF{F?sy{1_OF1|CdA;NLe(Ch?J3h? zTXfD_b%1`t8zb-4li|B+^XOwA0M;5NJjF9pk+^c<@R7{-`p{#ku^(^KlNGVrg>@eb z7e$=)vB!(p12R~@{{+P581-|T_c^3hIbQZ;!-bwkU25r?&0l*Yz0IIo*T=v0NLe&` zGTz}kkHi^dxTG{;36Kd7GbtI}PzU&nPo5dMUx9e=A`?`QwSMw6SV36#Hdnxe8{tQ! zcCNHE{_L@E%)rY2>r#MtenIV&i)*IFz-m=txk3rT ztx-X10jd`6rLWiCp>m>>(?2{I+X2WJWNSc$j$8}CDxAvmK-)%LpiPolGvjTo2t8X* zdq-&4VxuE*NV~4bhC4mHzflom!_K|fRVZXtj+nCF?CLRi2PlXCiwx`rWLz~T_**wA zDqu-?HCMh?(lSLk)wkK@?oEKyJj63_n|taM`azTY(-XkB46hQ5-U2$77k_wa>zi!l zsr2!NifmE`8ZqK#+yAw2yGuXbFOC}7-F?B70K9spvER$!NLV*)bnE2_MlmWH-sd6dmK>(}TQsuOb!tL)y2SxG$5 z$BfW-pl5^^CVvK3JD{j8Kt>6tk4P#;1nFOG+SC z?+klNorT|wJQC&+-E;h{h}DEYe~aYV7U$)S|a@R3Yu^K_gY<QOj4muKuT9-(j8*5jBY% z4P8^}*p`;7JgduJY8?Y?vgLsXjK!ZTU=qR7&C($=js>x1(gX8g5r0lGixqD9X5}SC z;*7|iiX88ND(OwK|8M|fl4(4g$$g=aB|(amDG5gp-^|PC3I7b?T+w^As^=!vF8lKjw}EQgg1Z#skjs z6v<5ZC*lQXgIL2LDf#gnkHCDZd+M7r(+cC}OM!YnxiXKE9%!OkH2)vhkh_ z1C@F-9dtDaa^+oO&Lby4#I?{rDn>+4UI)Z-lEZcP>WPG{h+H8yCAZxOX0_zB6}_$D zCIAx`=Y$#gQfZ4&F6x=cn42BY>(G(xdMkj{CDm9|OTyc?IihF1r~_-=4r1*~AD26) zKpLZZ{Gft6{T`sW99=zSjr%>5a1P84_#sEK>W9EPgNoGJNH6!E^$37*PdoC*v`QOq zC_uW&U#^HXe|+&0DRq6+LA_^cf8+A{lb#mA6j0m)f-sr7&Pt%S@sNh%pAAY0gfs#ig)|a z6S2tZVX|!ektbrNr5|4L8Gu!*IX6ho`qGoc*XHou1!BGB@OjYJ1g%wY3DSMWI7Q^CT(el)QTjcd0?jcke{}|pbLYQgWz#O68Mb*^ zt%m}}J~Yq%@IY=oPP&X+#=ik>9dX}0#IQu%2-bW2(n}LrLHto9PEr4&^|sLI>(|TS zML(SwH`&m6!MM&?UZp#LwFFL5stE7U6QsUP4R*w!9X-4y9~e?Bbkbuh!p4pH2lKzA zy?~4xHVim|ZDY}S-9)zEmT=s0c?q#3-)`psGcN*|x4i@WTNp1Y-?%e?4GHE?vt~|G zEG`uvHq{lM*#*ev2pS+ZpI^O)XO`9T3t6my)l$vdAfxsKF>dh%Ym!U$a$1iL<&^(; z7Uo8EWq*%=xv*Jw9RO`z3a3M9N<$jE`fnD2xBtt68E<+Um(fKd1<>lC%5IhZaHz&J z=`b5m_(BDYIj&nbNV^7TV_z3(1p1mHu_OjDO`~?sX#z41)L4c&ry}zevxHdxbmBXw z8N}2dGymD5S}k})%mk{gw)z0ej2PDhbR(v z3H2Xcug%k_2N?CZLy_9-ffH3a=xo~`0BU99D2q4PvO1uOZbl{yCPuBL6|;!i33$!{uzIRT)XR0jKGvdG+z|@=hb%8%DPm0UKsx=W@9A|Q zTqWo*!qccm!=TBZ31Y3K!>Wx-p0vgPo$c}RJg&qU)zCV^&_!F7VTxGiK{b&HN}7a{ z>NWWGDCoGMSV~jO$tR9>s8qzAm<2aL#xlzQ4Swpy#Iah1H{v5%b*!gCS>w_KPwgvY zGQ!@0NmB>Lg$i0q47tgkG*Ihtq~k~% z&xQ%0Ru#^VflT0EGLTGj%rA0$=q z-3i9$yXv?e|1&voR*#w#aAOVaq%5*F>20VZ3ia^_I;)~4Abl&9Hs7o>NZb<5>T=e8 zeE{ReZXW6Y>#LykO?`1;jFMI_eSS%Y!J7)1plgbyoZk%$^-KSU!;z)fBf>>f!%?vDE(_g?nX#}jKkR37i~I#am~0=jCEm%rqp4YRQIsM zl(Kqq&WJW}ssb>_Gp_P~JqTN3Zqq1plN6!1;sw$qFKywUBb}|9<=HJkOuBkTLFZ6! z@Rt;|Z1f0}PJdl5%O*UX*=JjbTEEksPDQrZ!J%qFpTKW3$QZ?Kl}UTC#;Hk;gA)v0 zyL&1OP2t^)%N4Rncs8rPdyOL0kFz%Ubq~UpCq`PlO780CIj}hhk9{oL$K&Ai%OUm( z1uRRJ#3n}VKmUpnEK`g2BZ_r8z4t2%2GnUL3z31I1^+Lka@{EJh&?PV-3lPq?y$nr zRtZ|qdyz-fOVkXLA1WX&zJY2u9$W>oPM74rotUy383JIXF>7VyJ*DHZr0&CN?VEpyXA>o|`iE)54h1oVFC49? z{v6z377pcu@#7Awr&YfgteHo66dawX;OL`03OyV%sQi4KBlunE)V1cRj)eD* z5Ue651r&Hx>M+Tie458&*;1dWfLNPU3-*sr0kS%?>~fj^3=-LFzy)G?eFUVX(3j=4 zVb4An$f}j*zi85O*Z*^ME^t~;+yCEBQw`~^O)kkTgeZjE=TS$prb?`yBU z_S(}U!24j-(Qgd^4BB9iym|%!tM^J%9R7C>h^1-I78LqjZUr(fP$A=fWp0ebtf4Da zAIyu9!krFUdCUVq#wvn@zj<+_C>mZ|qPMvAdl<-)Wo@tWOBA*k14#4gM`AFaa@0K$ z6F^HVnZ~~@$e1?)#^3iOh$YEpuH3({l5o<kesGB}MSwL} zmA&vOtyR%Wpq3k32klYMSd%7(^i(6ereB)Cj03KpaF^~a!d55_Q2dg&V-!>bvH~dd zPrnalGc`&HFzB`fZU^Gm;q|2GH`r zbpfluuA(wF`eW0?5a}u;+ijUf$@P3ev0dbrlFQaV#E2F7;VFY#evDK);RqgP{;?e_ zvQ~g*Uc;dCZ=gxFvxOb6pcP6(5F*mQU!XC@k+$|h5B?rwaggMX`7c4^Q}Pa0q5jnt z`)2ESfc8J_b7C0N=0W~n3R{}8BBE8EXbGa>CL!kcRs@W#pTf$Z#*;-E6we$6lC&~|M0@|edIMQS%AO@`yDmQp%=&dOOZStH zj`6taT&^kN^J5}SunhF`j*kfi4`*6b=br#(oU_lCRQjfnWUN;d`OOMiPT_RKcRex2 zfE#u1E~i9f7>75M;*x?GDX12F$HEA3MF(l7PJmT8i`Ot;Q4-Ujr|9zP3?Xz?S-Sck z6tL>V4z~j97m3El4UCb3cjH<=AVs*aDc$&JAy%ByngZWvP>doekZn&>DTr}WDF0ra z$_OXbjhGy?l@FJLS*x&Ka1Wr`pIilG^-_Rimi~T-fMiq$x2e2;=W7L8^~$>xGv#S% zJf8DryK`+yBE1cpn4_wRl!(>z9P};W5Usc-rd3s2Ijc))Z6so~_a0j!$X zG21kV*Spb?j1C0v7Ync=qdLy_=ZtAaAX)RlivY4+UeCX>KfesfN`mpFc2={l0KI4vY}7G+ofzYA|3=sOUZ{&i9JPf@1fL6vg}@8y=1BlDz2)}Q)R;ga<{tE{ zcGF`5K}Ak={YQ|Mrc%1kyBi2w1JVcV0>y8_tYVTw9*vM&XMtLoC?W&PubUm?qV%wZ z<=30Rj6p1tX~KPP0kGP^PL|CSTe7(k3OCLewrg&WP*BhSU3dqOaaN`KD6*w_WgeKN zj>qKz1N?>*fO*Q(_dzJEd~j6gH>Ker+BROkIHsV3r$X-O0T_QszBOf)#s2iAKvq&+ z4^Fk^%Rr9u9(7h{-1r$T>48V=Ly z*q8a;_+a7S&}N9A`dUnu-dR+vbCef^Sw9+v2f{;MCuppqGvf@zKh_8WV{CQv{0{FD zutfD(ouukzf^6h5gXualIR(&Ct-{^YH|wA=-dQjP`O80yVNNxw{rdHUE!XgF@eqIi z1`unQFeCXF1Zek(C%v0C^0!rw*@95~`#+0G;67E-_Z6SVB+SP7hc6-td#wu2fR6bG zi0ue;+|Q`t;;K|;r+*pa4hYJ`Q?>vkCD7u2#5XaFP2&K6fx^a;2`D7ebt{(u+yb~+6HFDA4rEGYWp8RR9br2Bfs)@b}=hsW(j+S{kF$+f&)fvo&#p- z;89F-K%S_q8pA6e{_zwDBB%e;?~%feu#ft0Oc-Ce{D++(Zoh1^2WO5PMBmr~s9CcI z`5}OoT{&v3b$RlDp40-PK8~Pujc^=e2R)OH2elFu$UwYFVJiVL z9gA+869IN;lNspGJBhz+JloxLoD`i5WXx-+XZhc~D8`p#hCzN$0TxSdmZ@$J6lCQY z7WK|0k&GFLOAMT`mIGPKpp@hg5c>oK$N_Go zfh+&LfMkRt&B}bG$Zs79XjP$8tKcysU>0*&3GO5P_aaKzmbRwa-+mql^Q^8<`u`}9 zREVx`lKGt%0$CEs&77VDAn9e;I#Y<%SHxHq`F3BbXgrV+aeN}xLO_f?&WlaQ+P@hU zJ5~DPU#=vcl`QdsLXGtJ6QFghd~I;=S2){jHE0@f1t-YzBPf;AZ$d0TJR@TFz;5OD zG$M8&!BfPLKSP!bY6QpoC&gHt=$Jm^%xN)g8df@x=ch-6!K<-&UPDBvBx%k&-vD9> z51`h5zlk8#9Ga?f4uPcA_09_oGyY~ED`Q?cHx%!|I!>_NBF_Br+XErVa$?~f+P04g!mjhas_{qul2R}*J%FbAnk3mGHXJS$uqHB0O zvoa#Og$_+Xl4?fJG|ayy5^GoN8~xyC2`0Z|!dd7?KNo>u>*Ei80U+U|21$317YSS8 z8Q;M;?FvCgQl%kQ{}N#%MroePKttCgk+Q9IaGMPHHCEKAQk+_DMWup z8&K@8Ya(JY#5Au4mElKk16uWj<5x}Bb?<^1S5#TGZ&28BOf(q2$9q7==uqAgVtqdQ zgP0J~zm65l{d8eT->pE6;s5?&OeqM?v~<6XK*p6$dazxjb5?DNVAz!hBf*b8i=fWW zu}>iS#4jRfM{HUg-WHhc0}R_vTVfcuYlVKmH-xRv>5*f(J!8TfDAdX{kmEe$=zv(k z&6=;)&k|9reRcyy{-muyNhwjVa*`oHGpjW2jsGNI-KLZSsH?swXqnK{DX+im$4I~x zb-zwQs}DOq(wtWGGhosJT!qt4w^$H@4KMxBP0m)xN~}kE{fJ*;gkWkyp_~2%WI0!& z>x0j;e~&~cXM<+?gFk?*ELdSjfR)usy+S zg@=)_&62X22EjwWo?%mU&^9YYYg9V0M*VL$#($`a1?)e*it9?*(S-B7Ecs|VC_ zVy;FYWJ4G*D~-(GIL1GLsM)R_UK{df3o>qo$Wy^13eyQODQ5rP_oVo%6#D+zQwS$w zad54~_KrH=wLh@&S*2MpT;va6(zFcWV4KlZgh_pHR?KKA^!p35@|YyWic#dp4FEO1 zcube)_lgM{>$*;@IbnGURqL4tX3|{-#q{NWDV+Y6SAtnb)yl@$8!E)o9-zDVw+4+WrVA2T&H=M><>h0EYs>3PP{xh#*)^n6otkbK}r819jpa7%udO$ z1gR;d!af$1)H4qCnUY3M0ZHmvJ_G!JZUC^F8d`+Nao7yPHb^lbLFlUt%GP~v1hxz* zite%$`u&7irE&{Z*Dw7+%^*!L)XwNzpt1tuklL?S&=|0h-fwP9xu_g_4_VqyyCY(f zv%_`Nn}k_e3Q+)ao^bZOh~P9tDwj9!2O%$c>x03}0|c#XT!O=Uz=MRXq3BZ#G0pCG zJPepE87q;x&xK1PW*Q|HwA&s9Gsc++@Fe@|V*pk#$!gy|PLRpHKsweJo*`hyAavbA z9%?{ugyH)};Yqud6qbkU4*@eSYswkp6IVcE8S&m3WBP>W0E|H96aIFElX^$5Qv-9m zt%`A|EhOun0*r=!OWlDuY=ighDZ(-@lhIa9kw)7UT4m-NAl3@VTFGg7hc_e0aM6Jcln)BD z^v%-FH@pL4Y!*n3aoc;5qNch`7HC(00Fv#LL%CU6=qCtDBC&b!_kT#(8YlHxj_C09 zM@GkD6MDw69}|`mj1}a(jWJFX!}K<2htI%`Mb>Efo!?)G6|W*+Mi1&_<2M5uS7>d+ zovx3+jCAqAa=-IefOH~U)o>Evy@DG0rV40!gkkq$4Me3{zcX*kcpM)F^g zh67=q0cBo!Uznwf1K)E0!>@#`(itEX5;(&``L9fmY?%u5N)e_UDPqpU{BOCD1}BsS z95>YL0Ai)Yk_;9GwgIqds|33o&h%jdM@nsD%y5=cJ*7|YaL{& z5Gyp3B(EDVCym}UVm2`{1?McyRIT|S8+&BO%6_`j{i5D2q6~};DZ6Qlk z=yJN>c3e!6Q>P(1Z<&8mOc4_nZod>!K{w_D8mkg$D*gEd5kTHx$Ki?qtkdf%8*mCzHJk%K($i!-e9p0;s1vJLbNj zA||7P;dZ}rIEYn%_6*5pAfT2wlNaliEX4+k)Og^KK;nNhgnMon+W0Eb7y~psk)BB- zX)rJzi)&*tVK2|)1E+x`)0w93GtP(Hep@Asi;i7~q9mu5kap|1(y25MU~`@LZs495nydt6>iA+B|@weW!ba7 zL#9S7s2tfQ_zGcGZN-=ey>LCjgeCfoG&0A|ju4YM{e$5p$nxj728BtLfTSOXvk?Em zoRoyu!%)lK9Ft*dGeoDP-`xRZJnB>_>{e@LYU@~M!8V9Vm42D17>VZgYrS_NQ9)26aC82s0m z3fDKV16A1CrJ9Y%9xoBJs>02O-8K01Z(xZ8jCGe@7t>4|;G``|f_L747)SCNr8>O% zRpJ^)!5xf${p|=8lxVxO=Q|O|=e)AmU;Z9|4Im^ndBRX`kADDaX+wq!w`yOz0mvAr zF0PP$!Kp%w2**|7aiG5l;`qF{#&`G!e_Gk}=1MTUxc@7#q=(aRyeuiel3?@7bXw+r z{1M0)3X7i!LoI)bgfhI77;r=ih0}Oep5p~sp209pWA&0>z-;A@uODlXef(eGRvz1` z`pVxxlJ?B2Wqo~2K+>-&P~M*Wdn7;}qEjprUd3Ls){h8H~5P>Z5 z+E+as&@z<+o*{nsQzMWc*4|@Ii-2wz^P-smYh$SZ=w{!Y4q^qwPC)x&f6kde#sHZO z2~SVW8yPVUh-||9=v*+%zbsGgo;IEbV3kR49OVBqHip9q;V~Bzwy`5Wdgw0L#`vk1 zff`R(8_C!7CCYVzfjd5#d{h{H~fDL)oqvpFlcFTOne z9YZ~=`L6&-O3%$=iSav*2Qqf@VUe?*->#0a@Fb3!zu%t#vb_s)h+qbAoIs05AD89I z@hKU1M-r98H3F@Uf*+~8KgTdfRJH!J`WU83OZ~Bv2wRcR5KyGorvO+@Fuucba4uLA^eWG#n4T>NzLHl9OE^Hmg`@e8QzXcSwSMw#F}ngiCi}tdF^>6N^55JE zVlCJoZ!-0F$6$5daR2x{1Y>0cC7PeUAY$BEt1i;k;s#+>R&yYyt@%wUA->6Cnuu^~U0{~Wu4E}0AX;DgmK%VD6c^JUh z5sPMoHgaiMjC2_aJot-JcR+uq_jhr-dZbNqC-N=l1}* zw&}$2MY{10HFA%BF;d0|J`3IzU{%C<>Hz=xM}&<9t=*{GI&A zSryk{X#KyCJSI>t`zW#{ScOdp1CkDsV%#eyEuM$^L73GaZ|b1%a0g~<13ej2ah^XW z1C-Q8xh$EE6<}o#0|A#SoJzC+w1mo}lRQsht1^z{wBp{A6A_Y@qPkTp5E_eL<+61^ zE*U=4Eq(1(B;H;pC~3g33z{y#`Y$6_7I0cF702*7&0}vxRz9B-Z zH`Z(32KH|XvGSl-;1&-aR8k@x&(LGu6Os%9jlacb5VoZ0LD+_%?df$+#H!^A0o}4; zmbk8es)aj_fy_!Eomfe^dARC$A#l?0^c8mS0<1{E17FzbZXX+|@NPyA{qib+WXzYK z$UA!iVPlMy!;YrgUl$``I>DgdRe(hrq$`QNQv!~s=(y8Tf&p@)IeTJEl_!h2Q-=}6 zsdW*{$0b1UqjsAxD}=09O8vVjP+cM0f~HAO*y1p_ro!JoH6|h_KS40QemYpvmaL8@ zv;@#9lBG7x!VRLcrbH>KuDo7|Wsi|1)!)7Wz+!RQ9cHQ?f~@R?MfqG`R5(c`Typnq zW`kH2BGF>u!}hJ+TrfsJ5OAm87L&lzpT`#OAZ*#v)!83?dJjOtYzYeq=j`(rL?}!J zY$5_=(O4X?ms^(7pcoEzeO>M+nsFz4W7fgjg;?60b%NlV;XxovNRPWz`|ku;EW7jM zPhAXRWeT&eu9x+DG)<|bD!45?PEf*9xC^&TfYp`^c>P|F6HaPN-h7Ju8X*=1F_q2O zhG%1xaD>_7MZ(!>xvUK34z=pr3blmL3NthNoBl7->4AHuFb$xag(h_Nn#wyutSH#X zF@V@AeEw>TYp((|WQKV|(?D=|(%#hJ3re>7kZ-QDjiB1hm zx_7{=Hp-=@-TghnHk7lL3R?LJ1RKwCd5S17NQgzm=`_dT3Mcg({Cj5B17v#)mj_DA zeXC6o!sH@M`mF_7G6QA3JNWZR!uf2Krt0orrVxv++WE+@B2+1}7N)9qw*pz=C^DiA zJiRB(SQvl;T;jq9DZ~{C4oYhMr-H0(v>M)NeaC--BxR{af95-Gj|nj-HS76@{|#jM z;ontG|4RK*VHS@gD)f&l1z72Dkx=Om_${WRn}#v~Jn26%PJu2r`MH^cvQ2D)vP8u@ zTLLDf!_~aL;hI5X-KLA?aJ}K-kK!GhUf5&S(=OnG(@I+zBL`bmBVV zsxDyG!6|PsrdoMb97hPX&KK@jhgaF|PQk3mTI4U;2g;-ZOEhcMH3PWlgVUJY5haN@P7_BKs60$ps6OIS9BC*fU58LcI zVU`#e`hg<9M38aWKb*P0b3z2LcmuEk$+mztxrG!kfaYW()_HU%`#x- zleBin@JPem(tQ7XEnzFH4tn$?G5+AI7(3OO3#jGl=@KqBC zCfsuFqxt6bYk-n5SrOLZLneY)q8M6H^CJWp337urEN2KYUU9QM$lsh&NT-Hy_EDNh zL9x6Z|MI7pYIxz%|7B7HaQedRuw+V1R40#5eQYlT`XU|*o^*J~-l;iS6%eJf&WVP2M2Y16YYUS%e|0!6 z$rqONK6wIo@D;*V9Bg5;r$>Kl0yXw|7l}41c_T){%Du>+`xaqilB11bY|qDk0Adwu zCRe5z820x_$SE=>aDJ|kq#0|t-{!v%V3o}D93C3oupuS}L@U`g?b=U(jd_#?oB@7L z*m@3v2sd4|9sl7gK&xq_2qZI(o&siB$fa{t|5}5L4MYoa$qKSNaT05|WZNiL!cRGO>o3g^yHN*~` zfh^_18q_h9JYns+kumWlw;LgwO*qu>_wPY4TM^uGD)%3D2eA}&Ramwo8w6SPnFGi2 zePb-W_)E)NB*glf6iV6v>s)95m=>2?`b!d8bwG?N8$0^Q{Re?0jU(@Kes-^z78|6j zb=)IDjZ;Pxx77Wz6hRR#LHuzc)@g#=5ytPNK7dvqIJe02rt35YQ(r!^wFNs#eCucMMT4v7FdL8j9y zsv^K*jGkx7PyovdJr4?hOaiYS24-!j{82RMm?EG?$wOo|SSN_mGOSV^`T(HS7+B(a zpUrQ^ALCm}BrJ7-l0vYdqs;@7WTi6rP$?DpcNI1QHd>Tx z?Zp7bob)2$q3fG3k3`bh)cTI&0gR*mm;`CqGzhYUus~#xY`iiBcm{|w?g=px%OUMk z?4PKQ5U=P4+wA~Z$@0*P7v%Y8M3f-%UV%f3G#YYDxCFV+%jXaq?B`4KDk-t)aEh9{IbPsBsIwqqhQc~>D zX`PE+h16j6ofMBE)t`Y3+9^v ztURWCQ-=V$S_y`h43ML)Dx);kq!Cxh_ny0^+Qs9G96>8_fviVOR7%}n5n*Neu`xH?~%%@ z_J1%gkOiaEKxg~}KSg+HV~ZBhq%<|mxJ=d8DnaWMLuG&IXQlxC7%RxT1Xv`otonN< zrQy9~hSVAXHr|l7N`2SuG0wn>;&NR^efBpXE1((zp|%>oK?i4(#tk&{IZv1hvi8a= zV>S)>brxvJ^Zo_z8m3{a11f!0OMbTrD!c1Yoj%$rrX}-NyR8<$WSk%io(9h zuPg;h#xpKN!)~O=&}@TX2S*D(CqiQkfsw=Mmr{WHM+no?s-duglQy@`!FvT+RRr6fDnpX)ii#Us54`E8iD~r+PC!Hdk`o$B#j4xc^bI;7r`Ex|D z84Ty%Q)0Z}<5I<5Fg;?@upxz%8fIw~Xzt7N4~WVtVi;Gy#N;K>qQATI1`z84 z1GyEw@fLzs6eO%d|LT+ zWh(UJ)}$B?&NTHImh^C#)%n9kBD1t;3wFwvHwW~V)OZ0_M4g^d_jd@f+JPxu2Go4K z=uJ>dTrVH-rj8%{PD~uzA`Y1@5nz=;*G{b5UwIEO=^)sSa?5y)kc7`9_197eMW?!! z|58xGI-Ru2cYPng_+k-~nqPcXeHartoEK0kyQpO`kT88CYXackmZ6CKuMi zY7!vJlLIa$vtvb*h?^J|UG|LyOZo>Vr?+p7DbNB4oIF_fPavxTd12Oj47L8=A0wSj zsp!*FNIp8SAoEYBkR1Q0bH6Xh>Q2{Vn2`4S3B>xJUPH|Mg;@sl z5*;f)Da3fd!vGEOT}mS1lq!GrbV!T`m}G)tx4_0=B&ZcKXq>_n8$hcJ+$o1s)LX60s4yyI51Q{0`BhsjC#sVaLPq!GqxrlJm$m+3$OgtA}8q=YB1|9lX zL6#XZAkwh^ScoMgi#$K>GQvrn57Ghd{Q`Dr(*bGD49S^JB2%RQpYHkB;F z`~yO)UU;kkhojBI{Qi6zE~vnD_LmALvodqwi2Dg9C8-nHB7gXz2(W#kRX=$Qz%t4kUgQ@nBWM|E|BI;s4vd$BB@$xrS7TPt z0k%RSS=TUg2PEkXvotPe^L6EpGg{18)W$WYL zjaf)h${h*!9*EU2j+~idPFn|Hl~u_iQvD%8%S_X{nVG%#Sp;1M(yKz29f6#rDsvmrUc1ozfK{OOss8bQ#yB`DC%;nP z@5dOYw4y&{IbMLJiyoD=veciD((up=JQfMa%BYf@=yyz`rwX?0*xw>J{Y2Or*|Y}! zLw9JlaVQA8Lk_u${E8e$Ndu@YIU$r8tQB_xHf;@5u z0OJAWCwuiygpCJOY^*STlMt(5nf$Xys`cevz^u^0yCeAz6=V^UmHrVQ5tQRx$o0GK z0b+fPZ6*7jFl+p@e@u)AcY>8~GlIfA+DzFr{4Y7t^z zu+V>`FIX}&B6GBiE4rk6Wq&|RnhwegN+kx&D#+f834-fYqO=ULtwJ;( zGZ4TsL`|pj#w9}{Vit*hSye=&Y(wP8tp>3ovx-vE{#S_9zf58Fo^bSC;q_&j+$9M?M+C#XIiiog~oug1ofI zp`9f009&ooyKxg1+)4!0N{G;{_X|sn(n^TdO7`e^O471Mj>G)`FPR8R#uyScFNR+L zU?ncVeY9VCVT{9NW-L;x{qV6dPJVC;lXDqB(!}8;#V@})B6RJc%(N#=i0PoP58i49 z2(r9!9m04XEWpx{F+azhe-dKc;Cq9&%CCu0Sag{Mf0zhh)e+3jY(qV2GAPH5c7FA8 z1EHk(rLoo%@Z~pvSu+)IJMh671X0HG2xg^!Z!Upk1Rx;l5xy#a>0J@jk#X3P7-$SL zx9LPJkXc@6D%fu`cmifT(s6BH!~-5mFlh`uYr_D(T98!;(;M6Jc6S3<`n(*@MPa`( zMa;9KzjRS#Og&7Otr%jD#FRKqmxU*Q*6YwqmiY@rlvE!3PKM3pDT2=jotIpfCZrLF)hn|2S#J?B@o1Z9 z(5-`ZdM~D^=eQ_AFCoSrJ4hXKX#QET5j4wQ_6?qu0+arX1Mo2IJ}1oTgeNA$7I?Lw zqza_Y<&b9+NFvms0trXobe1!}0Jd&7q)2bC4Eh$paxEIpT!IR;$oKmJFsXIcLx~E< zrBF3STPOzn@q#QZUWKL$tx5qjWSkwpDZmofH8Ip3Y{}lp>6J~qkW&slu@xXoiN_yc z;4RTuWun~1M2u6#Apb4PPpC_k)fpUpFIc~CvubOZ|z3d%76Yp8uWxC4}!)j8kG@R*mD1*prkGkh1tyYj5M+hmT=_y0a*ts!_61QZaQnq z2+d|#Eo0W6Lag`jk`QYEGt}qDfieA&R-wObU`$9ZozSgnSMhm@vY(Z`g@*LELbJ&a zkrmZXDZ)#j+Or%tDAHh#Q-J%J^M#Nd&wE$VObcYNwe@rmt3}qlLH@DPgsnot z(JXyo^F=^bA#yuW?7tLXj38KRqWVgR#X_wTj-bC2WL1O|6QoSKf8M1rtsv+5qRV2; zYXO|?cRjS?h_R`oG1yK$tt9RygeCYNF8J{ibZW9y@L?E2jTVoiV_0eEAo z510)L?G=V_gCij{FxwL37A!y=2o*` zp~5#l18$|#7Rnq?1<*2K_LAveGqRyLN3nO6nt>efn;zG1YMx*f+h8q$0kj0eHFNAT zMP{{xqB^hGf0Y7UO=69>Y6TP)hgWK(U$|!_i1o#R@(^(ED#F%u;T<;}H{H8B($Fnx z{EKS{TLW=38I!{@U;SEy*v$n+aDc29FltXO{sqvImz*6W${NMBYOKcJC(jLVXe14z za7%!GQEl$CKK?jn#C_F)y5-#rlF^T}qZ>qJQESQ>FF}TE28Pa76CQg?(_?Z7JJHVs zC7mrep#DaHF|BzgoL_vK5@B*jbL4J-$f}lwLdwAlQY2VmmHOL-SV@^}^P(HpYrg|e zO3E<`1BR(IfL6=M!Sn?#Xo=49;Q46z=e<{mHD4i{55Fi)DXt;@{p~RteM;+g@62J@ zgxG!PO7`I`KrA;d6rda)W{}kbY7Pe7DA8E*we(Wd*^=-!9zpls=qo%x@{uA0;WtUErayEC*u1 z&bwCvY#!9%27h(lq=He|#jz&`= z#B@C_`gKYq{f~U&9(GVnn}-CsN9tb}VtgTB>#+t&W3>l^TI)0OGF|Ex z2h3Q}F|wIpHzULgJX(Dw?q(SF6cqS>G1PE2<~~&>R(IOdYV|v!JaUaNtqH$8MYJPl z2lnO}(3o+KJk4{{6PYuV*^P_|>cWPs^aEtg&)rXnf_Mu@sFU24`eKx2>XQ?Qo2Nsy&xR}%c@c`+s% zro_haw>u*!!!gN!C|Bj+b^_N7)X=O9 zZ%_o#CPt}IvZtuc64g#V%;Xm=CoeE7Yp3DgEJofW|2@l&n&h3bEYe-l5R{ zWj3Y}sIO-+LN*OMeL-tDwVNrq$9qw3HK$WdOUHU2*3}Adv zDD|MQ$_zOh*vg?joVKY|z8T1qYV=8&#We$UHp7}hRSs43q!@Z@vCt4Jh6zegbw4L& zLMyOylq}P9C7_82yiC9;LpT}^Xtr&*y1|*F=&Zb)VF&Y-H&VzPo#G)kg;+hD-06hB|?ln%}bb_N#zqz%K+sI)y)Y_GfZbIOml<$m5OaOf;3&5=dVct zW&x&;ii2YMEQVPf#ik=ulqSdSTsp9TS`RYUKPqc8QuOVzFi~j(Y7B-$1Wgm4DN(B^ zutN3Je+aS~Kr)eu?!Sc?$N5~DD)G~fim@oF-beQjp9ExlaK}(jIKamxB{8Wg-Zf&E z8jytD;G0znfdCp~%)j>J!KI37nAWOrk3L|=8F@&yx=K_D`&=fK(9~gIgs^c8cX=-r zWKt2Tpu~X)tMQ&^g4@_d;;Lc88uSwZwydkDJNiIEYE^&{8*Tu)rE!Dqg9_G?)m^YL z#MPH#wa`8(#1+vX*c~9qQe#!(jqw9hKsP&T{XuCY9Z^f3IwFM%IKe{r_QwlKI;*KW z2d4?)8&Qw83P{?8J&C>E@Q*71t%P(i9f#mfMOd~f<=22`CsHyqV?#g+SW>{G3vNqe z>hTwKs|&}ISTd#7aG%BYs!cbupFFw2)K;yMGNC=!5!k0*` ziMY^?MD=$8wuPz2JA?l{CD3|P>-Sy|W8~%I>DV77z>0=r` zGn{~0zUvPZ~DhnQM>+cn0Q89Q? zhZ^!lBr2}0#44%8@A+>aV}rRAJq+Wrn=mT}8Y1ponRX$-MyexaHl#z0)G)m=7?$Zk zXRWOB3yj3@0~f%g9JOWKsBRvW_GNpHm#Jo16zbxLWU}t!CMg*jZ>gd7P8pQQL*0kl z(+3JphBej(mHr3;#!2v=P^HtR=R5YvmQHsJ_1=ZHPyvn=fgnMB{avJ%HFkWM820x& zbpo6rcfi1a2R;lvMFeyv$HpBbr&FE;$a4a9Nq^`JDibdp9 zQDvL52>rFcS%`JfLJR@?3kMOlUXhPeLPm>YLw^_zH~nTE07d1uRQn>k2%t&YMpXFss_xOA&9O0CAbC9>Q?7I zdMJCe=q!C5Q=pYRSBUk*@Yt13=n)dmPYIcswd&j}BuR}iCR^+GQwS3t9c20+1<5i% zuWIQr^o#EPwRoW&ZSw2*WS)87stsD`E||CFW`tZAQllj z5;^tq$D|QM`E~OnV8$atBs#=%L_|ea2F=@D3kg}e)wPU%-Y9t-#43`56FQV;=obYh z*>jB@LG1s8Bz!Y%pt$z0r$kzYDBN2@Y&7X|H`m-Wnl5@RMn%q{#I*()A+M6iL9E1; zKxkc_ofQ?K#Q_bE;v0u{EUCr2PM~p!6|1`EWFg5Ehz)U>zbTDp+ArB)o%C6=c)AQ> zh~>(3#Z-qnnw@zdv;wojaa4&qHjS$R4#$I1gw9q*VF!O6xj<5;P6mD%f$%DZKlazi z0Y<^4+_O08-x1+TuJ$|+3b7sCP*B?=CSfl#q6_ZQ8+L)(rHM;mqPpM*+0_h6PEe+Ms0m z#}twks?awmuJM2%sMeY-Bx$YkB2xOW8Gz5pYJcV-P-JH$rqXQ+TUOc%ac9H7aa1Ic zB<#nYLfEoFB-eq~SNR~;(dgGQ?olTQsL@q&OP&caAXYe*V;*i$jc-w81KH|h4ydin z*8;Oe!KSX-wzd~`XAhKU!Bd&r3)S{wLp?J0aDiIo_`w?8d^=F>}WU0wC5c%-zGeNA41~B?0 z$N4?4h!MG^tOtV)vPSC9SRCYgrZlL^xujO;4;GZ2xRKKPC~QeQ?4dtMPf8?ATh*$HxtO3MsgZ%pAc^XXju>B zq6BJ=|G}hn;LcQu(OG&0+?g$9Ir(1c8N?LOUs|1JP=B4hGPt*i$||z7gk`nRx0n^< z>Eu-%q^%&U5X3^9)2Ad<291hCnhu+wBE{)=ep_ECaRR4*3w&Sk>zh+z6w;1u)L` zR$#hxfU|WY;fOv~lg8Keq@r*_e~v(7a6koC@@N9iOCjBp(KYM36k<^bE8gT3VmFJl zjN2zcNv3$*3Od7^Q!KnBq?;ml3QNW%$}renv>qfWFg{8dn{vXWw0-eGH#Igu85mL+FL}V>H zG{_sf{{UclCKgRae$Fou3u~o1@`#;|%+fKJVjn;SG{Yn+upKTm%<|-zI;b22nK8zi zQ#33gGA<-6RQij3OdZV`+X;4&d-niKxYub6;!A*xS?a-Brl-h~v`X}RP@gm&M_*yw zADzjS8Z+miz1&uI;Z70mSGF=W!cum6r zG`_KGLVT|_$oMv=36!O);##eQgMwmzfDkJQyIw?$W10cM74!IJahP(L`u3}h9OyPj zQMk7nz@$nBVI^Z!mOp%Iso_~Z#kBnCj*{vEVvM8Qkk2iDXiB6tk5N^eBFGfj&H3__ z1y0tBtHbr15h;ib2!;o!p!}%IBeT*4ViCbsw;3q?66J=u(V{el3*?blTkL>YWlO89 zX(J&_Itfm4ac$(!Ra_f{GI-DdIE3d3wc=t_rr~>m5UV8#%-ABQBzlJmvG1-F!Qz5# zqksR}n7CZ6VrGj#pX=R zwmmHMOr6cB>5_>Hctf*|E(5iy=6PuX;q;63Z>+eo0!zJ#+BYDUJME^!58ay%Xfj^7 zhE?n@Q$%aF@`_~Py(&!%Z(Q<(cSnktt5bj9l2{G$SIb>Vs=fEiF);=oLI->_!(?dPk^k7nf!9a`S*--SMb&x0j0YW~$rVRJnB;%~OHp2pzd4PID$g$F-yzIcmEF#8e^*Mx zi4^-&s%*Y6tBt`mY^SRI$0-#%QO-A5Z#wi&xy8$?_GCa_miqq_Xxy`4Nql-$h!vamu=l** zNvY_FSWWnkgjkl6lx4Ruq%$KnVWE(OxnLqo1Ai*aSYaX!FU1{xevCL&eyy?VO_999pp<-QY)q;@ zk0^v0W$|P%t8yiUa6@+qU{chg;o49bWCaV-z1>xMSh0-uM%Q? zb||mR6!Hjhj72vK?pY6?8e`%5Tg~3@1`zrSn^`(*$C-qZdX?fT2+jT<_kvo=1+}t6 z3V>yff+!EmYmPiQ=aA(7 zoiNK)J)?GrZ+%}xxG|yPv=L&g>JV~}Z!5sY2QGU1`+XCFp%sap=Kc>rkQ6zwW@LyC zUj>>lpP1Yi8kl81zfg}EG^5kxhj`9B9zbKBhod<~pp>;~f=npbYl+)tpl<)LS0c$% zMQO!ETd8I7bRm{6lWdiqhCVwbDhzI8{JANjn;eY5%Tq+#5NhyUAyh`tNYLD~))6*F zk*c`(#~ky=hcUBoVHMdYKvo%yKa6Nppk|;c6r~Uhpj9Saj>Sl~(3&SHeXzQIAhdOZ z`@})SRm}z1%04tNY_MCbhn=KBx&1WE=cWKMAYH1X0IMAaM^H&!BgojmT0~;3^k3_K zj!6wFlD;Vb3Af2X5d{mVl~%)soeca2Fj+NCgMpXZ0)s7YW)_N@u417&QG znZ;HaWCcZ%z!Qc4H04g0i>myWX{@lcvi|-gh{3Ac#)#JcA!Lk-paj{a)Q@(3v%RO7 z$%hL2yfa9WJH`O)stigdI~`wWn*1Uqqu4{`@L;nbD|BL)7pB=SQ%da|q?G+ekhKyg zVN~re0<3(H(B1vNg&4O;JNU-cJJ|d8224hZHgY;Y=piiIHCQfj8?L430Kt}G0SCSi z_7sxs8rn!4odR5shKpka*x)a&<=S|j!j>H?Dh^K@#_G39)KI z%&hc1DhVg$k^Y>%_SWzSA(-hf+Yh`jLb}h3TP1^x|7!NXh@nH8V@Si8x4`QlMKwC? zaam}<%2G_C3a8XXzCxIlNvol}q175_+%c^MJH#r{CF32deO)S`$DJiKTMQLH-;WVw zm04()1ujpC5D-yT7Wum`j=9pmYp7vLdw4@kOfI{1VW77lWy*Vb8hnFA{FJBB|HQ7Jh37~=k zW&?=*G)|}G;w2%q>X5@8Rv@EHYKUEf)Qf|apm7^q5!Cv#ge1cZ&qbyFIRO@nv5jO` z;^#ja*-(21Gqy!*z>*HB=h!7x`2ZSk_<&#+3(Tw)U@L|i3OoB@eu?56C!Dp(PV-?Q zF=M@I=$}s^9?@gGAdLzN_xsRcuXvk4wtln}qc}PG3ozq>y9H>oOn($$Bju!tQOn>v zR;0$m0KIjD#|j}vMXO-L3lk>^vy$Q}iemO>QYc8H;*$`^#lAF+D&JVa8$p>cE0XTZ z=y38s%^=Ppbg%F2&1nXlL~x^wW8W4zM`t;~Hz%SL^H#u&6DDz-8K@A=!0KU|?Ku`V zFk6>c>C!eiN-#!dO;Y5R0*y{eMY#?SkZE0lJ9)W84qBHsqP5!M43*niewGmHJNP+Z zb9_dt2xu@2!_5YO)n`#H3s%K2+!gH$mdw7Wk#tuw%)S9PUbvjY>2I--8qdfP>ge>Q zK$rAkE@$Oe`JI$vQVJgMl{&kvpoCdQCBjGN6oEC}soGaa!anwVL;R@y0AlOY&6#8C z4~+?7o2iQ+cO02gnHH_um*;^QqcW1h5=q)A1iAWPDDwtQb;nWE7irur2HkXj?$gpItwbRwtTY9 zUbudj5S?7Umx{exo#mK|Qj=bo+ zGtW7HHHj*nFIJeh!=SCj8qZ zxO2bn;Kp3i)}7zGt-J91w(f!t+q!YZ?cCTs+qsK7c6JwyZRh$^j!W9^>@I(3XLs4R zJG=1*b#zyhc63+0)yZAirK6kh)-LYqpE|k`RkQ{&J`6qS9q3l18!>V`cKKYl7m{i;;|Vw=-6C0@cUd>c1f-)EzETl-?wz- zjV;|`>b3IFR<7#4Hf~7AR<6dian+S=+_1NHa6>!p05K$ z>>ZtZh`2q$XY%*qd^hyTadY^)@k8foNOK~e)w{d8QtGB|G-$V_VDzpEf7sp5l8cpStxK>V`aCY2m&u9MxhjSUb}F?)*_L>O1ASN!v2+ z`?qGaXsDrHH)ovZw`rR)G&13KlzD=4SHe?0-*s~{uJPE6YwD75GhfWO>%dpR%R--V zFW(;e)^)I##_0iO_m76q} zGLFxsvcOlicGvgKb<=w0x+!_i-AEdX8Fwd=&h#z0uAwQ{-3m7Q#auU&yzb=hujxyR zN02{#k}h2LAfJVKtz8juYWPef&oaJ8@L76kYd4zjzrthVuk?#UbKFb(y^t}!hVPkW z&OJ{2#q^UEd@nz-wVOt|*UzQTc7vw|=bi_jw1l=<=iJgvu6vE}9Qx*@c^UUAX*9KM zMIUM9>cOT>&A92SGj2S2uAv?th1Z$%jXZc7z^4hU?q~W3bd%~cZu0yLd1Txxq3ANpBf>Wd%q_oF*z+*f=ze#P$xwQzI!Z6W2J(#pB{{QVifE#>>6rx;6$N4`(- z-S|87FJ#-72Kx))b)uceSl*E1X0GP9 z{o!qIJ{r%x8N1A-)94FzHPmgbb3YMpGUb~|KiEv%4Se1po%)H;T*`Rbn>3a&{wL7%5ht%&-z{&xBL(17BiZ9qp7WAcd zZhwBSCH;ric5u`BIrjl|->{&A%K@L$$+>&@?IW;xgwN#fx|JOm*B#tC_?|qggZl-% zhw`PL_2PRk!hQKZnD6@eq`43E@s#R^{J$d4C)+abwsj9S(WYxCBeb*YJGhw*9h}B( zZ{nQ@b^_na*Aka>YX}$dnMON};QLbeIGW!Vwt|=Kw8=v`ZZvo?*vt~zc^l)4@Kw;| zgQ*`(Bs`u^<0JI5SDo8HnWl|#ZX@6Is~LmT+vmboW?VUO<}9-QG@m%tgr~#DBL2Q> z4(lf2e?Y6T{|}B?*%xdo-bD=V!VMr z#b-62##PJ{tkVt52aS8GFC!nUp-mY#jVJIqpT7%RxXESAz0glvPuuk(Z^~FtKbk~d zmyzc>erw{hL21$_r&Ax(HyMwTF&d|kPG!a|EXr|H`mu&Xw^g(s({9Br+&K8(Lfv0Y zT2uM^FVum`_%k$r0$cbtZ4Ex2Pvi8Ku6{ge^QqTwPm#WB>6)rrxIdtu1~!FpGK=q4 z^yhYb8sBQ~8V{vBu9a&#y_NfxyuRbpct?BIIpR0cmrsBOxuNN9zTfNMnyxmi=~!qs z)9!pG^O?lwJ>pK`TQWc%={GE54Ovh9t<81)2oL5{qI{3axSye|+W@ayoGZ|L`3Jl= z5#Fvk%yna+8_B1EGBzHRaTECaQa+pDU-AFWcijkRN7HXI@Gz3U>3el0&h>$Ack1e4 zXe65)%HK!usRd`=ta}l;sDeBvb(4(3`n_8&b0PB?{b^cH=EFI;?g#k0VGMm~eQP(J z?}ndqT?uh!vv$v1&w5UJze3ZK^nT-e&H~cOBON{($7{)B66w6n_j*1H=OQ0W&2^Fq z8gHWAH@0%ix4=8~Ttm92@fpeIJU;&+y+xyQ-6rBSuE=o@)#SP@{5_7}w(*@|uHKQ) zcKr>ua4Gu+zGwDm?Q->9=G+qanlz3&K@OOdK|Z0L7JyCTd!@cRAs;Mdz2Wc1O87y> ztEb)S)>4;zYyPkMgSK0jar2;=$$HVy*SX6{KlCx`{`y|j7ub}E>_y;b(p2K|cb(Q2 z=8N6H!u-F1Kr_v82W#G3>C(!tdGd-UU9Gwz{0 z=koY1JD%&e=D6FTna^h?ylti5EadO{HH?48`V#)0#NQ>vnX)z~^pWeytCq0pxQ;vq z^Y^ro?5X%3!^gw7=IW+LOg_*$-HvoR^4XcsOzQ0^zMtc>d>!TFdjj#+5^uTsc<+oW zB;A30-d0*IGw%POxfGhki=2CxzZcG;OkJ3pR?^OV7Efir2>qndgrQR&KPs(n_jFCa zv;KoUy41P##GS+6jW4jSzlHp{n)HZsD|V`Ub3N1#AB!t=$6R zY!*#V>X^Tm@L9MO`ct4s7FtR8MLzXA(*OAm_**U9L(BnP;7fUEK5aaUd`Go*+em9q zK5s)ir)S3fp|qR2yY+|gn9Up`uUCKu6X`PLeA{fLa) zMqN$edkf!9>?s?+XU@&+=^7?vT!wjab`5LH$c)?0@AVt@a`nEun@1k?Bf7hp_1)dH zyzZ2rb~~T7p{%>BU)0_8Ag{?w_#N7rZ|~&}B0TM>?yODxK93LeGPFQPdYuW-<>gw9jK7X*k=kJBQJCd$*9_|K@lpQOE)?ddAPms6gheBTYe z8|ClBrzf9%_|&bW{Cpt1O`%@;fM4IhSfWo%XAC!N$+-2TJ>w7RZ8P&c{OkzL z>`kryI865?`*KcLq zF4DZn_&l9`csFzw#D8eJbNlf7f&6_sWjLEpPr{PF`tZGwakCG<^-~&yiT@Mv-^#dN zV3MB(gCEJ?d3>KNjCi&DUBYMDHstyBd?eS;VcddULHaYdGv<-+r>z9%?|#H<9EF~M zaX%B7pZ;&RAMFIL9pk8$nu=$F*~d=R5b{QgFVP zw}X!LaUH)cehQq=TII?7QNJ$Ny{y0a{Y}2}sQz>KUfST?7{2S8prIWb3$?!Q<)-}1 z-bnPr3768IjP3eAXlK$qUv&fRMSM5zu$P;;jX42%s0n#!FzGfh2I^b2b|w5x_pPht zGltJ(+M#YAexv+#sssM+2i~B1fUkPi*Ne%s{twcl-sgeO;j^4i-9G$Izg$SWtt9*+ zpS64%5gwaPY2lvPGslf5?lkt+>%rgV(~bIXny$XYzGFkHFs^n{J#TE`Hu3v(>ZHC5 zUib_q?iR3Ne9!5Z>yCkL`kGv~i0}=_EZg|)F8H6eIoDkZ&-JX|b$pJ4Zqjyi3FreP z1J|KLP+z!yQ)_oEai`E18=${|HGjrDornNngFP4shxEJ+%B4c_4ZHz3g@xOw2kMNns z-_se}4Nqm<4XasydwREu^t!42uV(yyh@OD{Aerk6c$xGE@pp4>KYpJ^zu8-KM=>sX zw{&0f`@LhlI{<7Z>q`^*mqy0_@{)|(2R5XSkRIs1dEQ-)nyx?Cl%Fa0LBv0_rF$9d zG3M`wCNe)!j=zk_xFM9`N#up6deT33%(xb${dH?s_fI~Lx5~IDPRzJfq_O%B^d9S- zJC-z7WHN3gpJ(};Kpq8ru5aty6Z3Q2;|)3P323w*P}Pj()0sTJAiXd7^yRmPsf@KR za@}7SSRX$8S=ab|j?3k{FQ2!`Yx!K~dJy*sz8~k)m%oqY za|)m1`5eY)JhTtOZx6ojfd6j$pfjq;aRvOg8-IH~Blo5*#yWTF=yvXA*TUURJI`N~ zad$Q_PxHBB9^;Vj#=Tj0Tf*BZe4oxYmDYqVwsBRio2ra5>`B|9CnmjTi8Hm8bC>dW zEuZJ7(l>gd3+U0ojV3&4pAK$nPv@Q_pQ+3hjjz&A=(tmv6Q_FC1j^WtPvcj+ zxuz#~a}SKjxc-#?{<0SAmpi!oYBFvC@&8PlUPK)o&u1%RZY+t&xz!_lJ9%3$ha5zp3{|#XR%kR zZ||<3+TP8bNuOwF?`|(-O`#r^tZ+0g<9sXgHuNQsZ{~YyZ3j2G0h#@%cCNmtJwLQ} zTfi3`Mg5JWF23*Lx*>9`drJgfJum;3j{Gy}gD;@2o}#by z;`e80(>lsfPk9;-Vmwohx+cQg+lO|VO#9W1rwx|#y^Zg68MlP^h#_@q=jHreN1JQy zpES|ATloFfP4Gs2&Eq%3lRD{^q+{yI-?z}mZtaC$f!{>aCrM{1;ePs?zA*{fseK*! z5pNUnF~7~{lSiE@&yjpjhNrMD^`w7(&s?%B<4UQs5&TxmXQBQkz0rK?n7ixAYaD-% zSD5_kyO9rlWF+Ax*4{CEU&?0!pXtc>i%28v&sHJ-vM;D0z3Y(!=JR_UW4Qhg>RIW+ zXMG90GUl5;rVjXC1kD0Ih;DUFtiQK1ugoI7CirY@OI@ubE_qC5{3(K2L$yQzzh)$FxKbM>{_W4@&a{bSKc>l4;5Ofv)K%+K;k|uUlDD z--d^Ed=M{Luc@a!$)Elv9oiD&tFG&i@3c>AI-fptD`|Ax5q;_|u5oyK_WAUSF724- z+t5GIW3e%6+_)L-G1!nm4A-DY(c3wCJx8zSC{Tg28AAtdU;+~;gbZcV1_dfmhKjiJ+rsuu_wBC! zFdzQ)JZtTJ)?RDx_4Yh#?R{SI3=h!fio;tS`XzJysBd`l7(3qYuS)Z^;ScNMjp+JM zc?Q3iUPDmt<=32HpL?^Vk2vy58$uFrRrVV>UqL^*$>4QhLJ^0nExzG;W>Z`>p8 zdCOeq&5Y(P>gzLjR-JS1rdRUAX3Uh{bew#4YIy&fQ^V%dQ^T7F^27TtI5)b7`u5q5 zztx-{z7>`Gkl&#IukDx`<~tURaXtOmaMwEf)v!m|Hr;R^Zt=&ISVI*qELYaQk*0ji z7Jltse%NyBt6@HUCbk^=s(G2?PvB3{IN&*}3n$x*^L+j9UyjCmd4HL-78}{>Cx0lO z&;KOf|N9@3@0$P4{%ZJfx94WGv>|M>F@2LU zBD#MCCrS6HAR1%TvrP+OtnYdL(&)**`&Xb`kx#Ja_-Ohn_g7nv`h)2AJDeN!=~3O< zM&E^1uG_Ol^Ff|+=x;vso5!SYjEejl1?tL=uYNW3ac+MM#(2N^pX6~3nHPoSo9Vpi z8F8DGbMtdw4I6}CaeOoR61i!l_|abtZ|-s*HiR(Bxo>Iz@BiMjjlSWU!f9PS}qk3HE`6a|h&oM>5rtAf${r0?bPC`B0>>fnz^_b(eXvPt| zS*!j_<2`y$I~(}v`D2}*$Nv`PH>xMni^jhnZ>Ik?FHHQR_RH53jbU5N>vuWtUn|Ex zavML$#V9>#MB_?NIaXNHimvH*yIk`Pb?wcIe;0OVWQ9HQ{nnnp3pX6!t6WiE@}{!B z=iEEuqV{{5{?;A#9NFWyx+p(a^$CuD6rJ-?cz7Mxi_SmhyjxMc`*)Il@6&Y7(rb@o zhZ^H)ie5)1&S!^)lg4U!kNS{iWA!2W9<;jmC-DVk%aBGk`H#Zyzm^&P75(Jpe05!& z*KWQqekYz2zX6++^W^Uw{~$m7Sor-mdQ-lR%~))XP~bX6uY{2A*cM}cRKK~$%|G4O zsb3ARDPJ`HnD%XY&Hea0@;@M|zkA52fBOYl)S(S4|G$ln$;Y8cKEV$2%w8L{k zM{#77&tK8EjAfI&BR}7hpR@9#`~?~280IYRe^*}K&kWChn%?`5${#0|WiPADRE;7Hf6=l9?KtI-+st-;?k2wXct){xIr` ziu^9J(;gL9RQToa8_#+Zzd%v=vKWrtsVoE2V+ zp2hE_Yy2#3%L+BodEdzlN6D3V7qLgP!YQ)yPg&M$)89A;R_RI?y>3puV=a+(*N}S?T*s< zG{5zc9>q&9N#AyH<{`~!ZC zca?phdqZB{#P2A6^E;mP$C=@m@A6xJ+i~abi_S01jQk(r0R5MrWQNOs;}vCu^B7N108(_^89OxTE0_YNP$JCB7!!nI%P72@iBwKm?AZT<2E>qO$KQTDw1 z>KXw2&scFSji+_#vmKby_3jqQJh4}g3d|F9(^9Aw*VeLFLx zVnj}-O_d!^AH9t#Ingz)x@_R2Y4 zt3K6#!`>0+4~o*%A6|A}>ec&bd>tX~v*V-aLy(K;nnka>zBUwHYmaunSDPE-Si^_z zt2CqhPKeI^U1n(blRQOvc_%ZB`!sH!-#0qOKNlv6n}Y4?VdGb{H}^ANcpB;*pYf^u zcjVjd*#XR^e-qIf`ciTQet`dof0$`JCAVQ2-oVdr9{&l~aTy2kV{@U+_cB5iy&f$% zf_G7ZO$`t0O~<3|uYBDSXU|sH;aT6IcLa4^9q4@Pi4fTzKig&w@uYs_hx*u%6=qDa zeuX)hhXpv`UUj|xa(MetMpz`g1c&!22e}N(u@b-9rp=IRQ9d&xtS76=L)b_bpa}IN zwPA86cB7HKmz>OhG(IEj72c2Ld*%WC`Hu#x)53>QiVAG1G4{NsZ1f)+rwYHRul#ma zs1q)<9=_?K@pP>FsUJ$KTP@<#*s3iRy`2^6jlD;Ok4N#!n;x1SlI8@7^S#0eajnMS zmRr5T{%`$bM&V_CLvxh0V=3vjAswCyCE9Ljzh2>#^9zSQ70!_7Q0#YA#_<^9&Z!pSjSuc-Tz)-Wz?fxn_l%$Uir9UMTqee+XN?xIYxWkq%oQ{ZHD+wXp4bf1B~zUR7+AMD?UhM9jK5_|s7kR$EO z-_HmG$Uzu_Tx`;gB7ag(`xG97LnWEvFn@UQbB}~y+B;WbvsS4MSHH5zUzRa6l$%&q z%zQdjW;_w9?)+7V#r`V9GoB39_r4Iy#`g~8sF>Y5R8H?5s>m3dJ-(xNsNTY#zeziP zs#h52IukGnQ;?5oXyV6jUh_z3+3-kcCEJ8k{0eDn1MR3O?`?gqcc@$UNT^2x$8rM6 z#gBx>`K0?7orO6ia)N_$641WhhesL+lo^YjB`OgK(fYu9W2W<55d07a2Z zc?Q^orUU%xG4-n?Bed#U+t#RG8?+U&UARU+QQOT{K4NS@Lqgk3X`^JL-))!PPV7d# z{$MYeAd}cnFH*O*npd{STkAyrd~G;|^oESkzJ}lbhBocI!ze{XB*m2=POn_2Y^z+K zo<#Lz@%o0?d}T$8xHOL9I8LCWDkGdCJHOLAoFUKQ0=mB0J6s~KAl)z|TqCdJCW>Aj z8n)`wZ`1GMJ|5sRKS)$xGpA}Z$iV;%LfJ6m^mDyJ1u9X6*i7wxmG+LpFAfVsoR^Cc z7=ENeD9qQ45#0~SW^TrxHD2`5QkBUBaTLdK0;g~WO$_m7#&!$Y$}nzY zK&N)O2HJN#9ct#=AFwz#)RFbwPlpEQC7hom8yWV+>z)qhq;mn6a0R8Qr$Y&@(JS}q zJEc|hnlb(B`UK^U50}U2UTkJW&(s&7V4?c8FE?EGo5VKxo|_SF(r@D~TE=R>BRzBN zI7Oz>KG1sJY~`1?3VEy~tAt|*v{Cbj>dnSuZSlU}KR{-laR+^og8^t#mzq~RtYfYE zgf`)na8$?I)$f`T{ZCB$C+k!CA9_Oi$vx`cV*Q7Wor9z^1i2W2(qZO2p8qI%FA`hCU9}3(Xb8#Za{>GfWUS2~&`dX_$don1jD;|MP?wVF|XHyBB>sD-?b`D=ZaW zhQh4u(D}uTu$RVYtX)~bZErIeAZjN<|% z1Ww@$&fx;mQ~QKVWLM!|hlt`snc<3X(T7=KE92}My>x@;hwJppsBFr3lYSf3-Rc%% zsCrJGjFZR3-^G1AKysV@Xh-jm$(_=wpJ);8P4A2LJNhHxlzya*%n>&LgD?cS7=cmf zDjXKZkR9fQ!E2q(ttbJ2)mdl}|P zXC4+{5lXk&|FGkcu!LS||6+ykQu;DfOE)eY+idZhw#Plx|CbF@{*e#oD3xTDa>tY-F2B|CTI4s|u@k$o7X{7cmTDYC8kzQ>-<#%7}ud08ZdxA`&@vQb=k{$L-^B@kR0;Pwt zLrEe##OamqWrvD;*`dg$x~)Tv$-}jc$=b+xZRK=!sB=#Jt;~=lTadKQo+ghX(P6%A z3~yc6D;yU-f%N!Zp*_#sUz<#!Y^3%#Q60_G{>ZAa(%xnKM>S6Q%^94-1yq^8Um~yI z8k+X?GXK)XOI&A#bk|5%+fS42!Zo*gh1z@eHXPwMKtrv*A;k}nV1JhP3fHA^6Sr{} z_wfLkUsZq0hJ>_DKOJvngx_qv&HW4ihj5Q;?5on1Na7 z`r%)P^e2ypIrMp0fO^k-5t-QM`O=p}@!IoJvPE5JovnV%S3ef3A7uMX&!2;LkmJj+ z94k>Ou4H~rSWU0ooWpj=32W)=QN1N6#P{WRMmeF1++>a$txHAwh&MXk^;DlwKo(&; zI;ZvvJIRi3@#jSGd_`N^_#^21Q6`;o=~TRz6Dm1Qt2n}9M_dEdcXC4Ehx~`mJB(6P zAdWgD(RAeT(2N$eq75me(SGOgP-EXg?ev^b$7x(oHY~PhLHUwsMA=-|U+DVIt32TP z$`_-@$?8;2XmO1+j^a2@;1tf_9Gc|6S^isMIqVeo*S$@VX+-n$nq#hi()G!DWosZ4 z3D=i)BUzuF6D~;S60YDHlKq^6>-5$>($2^UH|e+0zA7iAg;V0&`saiK_dj}uJ?o$^ z+56^M-FMytl+Y`tE7xrME?$Uy5+THet0yYweB~5hwpjanQ62TJU!{6lwL;yjRYy^c zLjKvzubFqCFLE#dgD?b5tF%A#v~#lUtoCoUohRi~ZR0?G51|c^IAFwJ(C~Xkh!1UGJ#RWT}3w z@DqJDJ&6|78*`SCX&l9IB*i6+y(j3chm2vy*HiQ}XlFB|W5zDDCG4wL|GMmN`>g(@ zzuhaGb6(LW>f87DU+I@{1=nyLozG{5n`Fo9?Dvx1;Wqs)?&ASU#g#}q^EuDWSQyzW zz3F{XEidu0#z<7jV~)6beue?$AS7o#5{8huNZ60jf>w+W9)G+zZl~;<1hh}(9{3d>hCZ`xXJx#b}xFyd3kQ5f8M|9Z);8) ze^GbV`0H4Ma}v$wALZsB%2}N19rC3!4TXjLq;EufT9khvKZv+WvI0eqx}QH%CpXC3 zDtTNduNl3=9Ou->)KhW+lI6w#atRXRTUMx-`k|%5%aB$d+b620FR7>V%|EpLp7u{x zq6#s6K>UCNi?G4+1}y0bUN)bxJll| zUED|ETN^^U>PO)LJ#(5q80n^0LtnD%iHwj#4!|G`K`uJp$Ot3I&gc2#$;fvbtzCC* zQfDXIZ!K;D?iwE^ksUR!%8T=V_oFaHcogzc)TF-sPG3r&fmxV?d02q1Fa9VjBI~zp z2usMNScc_D?%2S7*br9ITc>XfEm%!oi*|Y%srehtMK*d@WkXmmej^G{gwoj?LJ79h zE4OS671&ANjcR%vu{|3@6{7y9e*DJI`=8{kS3{w7iM`HE40r95uZGssuRiQYe;4`X zUezzZYM$__dBUrq?CPtb{LZVPqWjfQNmku>HN+;mM&8CyP41V@K^#UYDiB8aman+-#@vG{oD&z7{{ndl$zvJe3V98Q)9U4W>LsoV-$Yvd zY&Sl&iA$mE;s)j4p!^&7Up9m)Z7Y_c{+p*pbJg2^a~JpV06l)Z|3yF8Upqrnf9-#; z_CH+vN84HB@BY2{>N+oXhfQSsd&|fo;8SQ{4d>-8Tv{; z2PM0-%Wc{vimdByeax7><6*n35*Nb&=M}zd-_iTo?3-R;kZ^)b-slyE&~q^YEo_0- zm&_YB=%2_mqCLT*92){y=>fUxcp98^d<8J|iRSjN(4@&fPOvVK+Uge@|rdCDS9HXbYd`e)>VQ8@JM@ z`9GxDCcd}0wK$AY9LmTF@AP*s28VF?SXL-AUX>f8Dn2qkb{lhU7<o@ z=^EqTbYm*LvX=dQ=#lW#_f~|oaP=+r_b&GLKK8fxJr~!6w~qWE{OsiS!``v~SJ<`q zTnTLEa8F|ImAUuZEw`|9<$zp&y0)_kI)#Ui@D8 zu}$YkrF|Sl{nv!8D72`#Wyj0j-TguM$+6Ypgkz_$^Ww(v)A1X^>v^lfjsvU0?!geL{K3z))tbrV-F1q-;T`SohH#U98!cx&my4d$ z5zh(f@mZlAZSHdlburI1;W?AFC7$a6&-ISyOi%3bY+iYU-^@G;<%pviRfyiNxa%7C z@c^068^h2SIT(Pl{@I~?7$3vr>`;lSvDqQ!{>5izhwA6D!yxAjK`usM6vkj2CZK7t z{^u3_58J*KZEXD%dhCRR`9~k`n4+#j|AU4#`kxK@AF^?UxZ12x81zG4N|+?QDM+5q z3W<`ekWX(tnPm*g3e)H_(B3TVTGyE9T9_rS_)b=sL(W6#o~*ElT!NC0Ed5uO{!8Cg z)-M#j|5%9M;n(Q4&S7p8LuCJVeC3hQ`88`1-+IJ;D((HE=XKV4ur!uoIaXpd)?z)n z-qs(y`$*VGFF^f7eI%JkSW_Z*B3ffgf6VTtx9&0)@C&5o>if4D1IYA3{r`=Ap}f0a zsJPQFRIXqbu3;B$U@wr>!VCO%FZSag4xP?i~v3s(=8H+FA~ ztUAK(o$Y?jcfU|S_wjJTISJQI;uO8;vUyAv`*bn?fbTZ6O_aaM@<(s?UPjq`_Wxqn z*uzeaNi!kslxvXH!e^v$4i|6`SE~W`J%ijZ{~Du2i330o4D8-d0m$i`ijfJ z01UzqC`!j}ab+2`K!bJnA1hpL{$_5$?F$FBAqo`5JZgNq$yw(=Y?G zFbDI{_3OvO0&)?SU@4YiIqHqEE6K?Bo?O!_tfsHUdPF|>R!S z0gA94JFy#ku^(lt%+J?F^8o9EsM@F9AWpAFjkez7-q) z1X|FFHl&b7J9^&J?(hB}T7T$rPY$~7VWgk19`SDALveq8%O^nAA&C~GQ6X=Y=<&aQ zN56H%yx=|a1aqL+S$o)=Q#@VWB~PGKe|3sHgOd5a>q1_@C0xNZT*pmx74r2|WrdF4 zsw2XkPiBO>jw4m^ER>&1@KjgWm)KRpdEZg`# z?4gZWIrgDV9m_MmPc*)d?j1^9r$B$d`KrDNgQPJ6MMJf#8uxc3`}df!`XoO$*=Aml zBGba{WX(eMFY2UKkA}(Y-;wNJG`eo_f!<-1^u}NuCZIHB{Kq7ErQ;QtLeEEahrD&0 zgP;o2#MR5+400BdH~90(c}U1_%MRn4`N9I>MM$qP@7Q1-vf21XmdQuCd{xL-C93E# z#MddmdB_sKS&C&?j+I!AwOEfP<9+i*<(IEk`DweV{N_k$GV+zw^i%%+%8z>S4R@5^ zc}cQSxVV-Ng?+YJ8;$nZPIA9X%a#9-H3R8aO26Vg=g`wHv;XBWK3*Q9ydLWvwu`U7 zWv|i85B&ql#pZ+Jb|W$0v!AJ7!d~J1NayL7hI{st)q#oj4v0I9LVaX<)6h^#?-=Tu zWM@!@N5_RYy$;cMlO$V^#!(!{2^1SsPmv|(?O`O(q4acSxIkV)nm_JJi9^Y!9;K zBW;_E<^WOO67?~)v)Lf1XPeZGHTI7*2bgT^?{5x3FVkMj<+B2nS6#bX9tLXb!`$nB z8R52T+{XiC{#$)8`XUDd&}5$7+{b-(eyiWLjdx$sWB=dL{~HrNxBoqxhUwboZ1z9d z=vfrKGtB-DYp~Ml`n_+|k+~QVg@4mK6n<}L==hU1BHUTSA4Nv?!MG^=ygD_QokE|4 zDcJJyGsafeFdxVlZapS%-5FsTeFoa!leZ)CNVYkz*o*CVm2H+|b1)C319L-(k1Z{r zS7uwAwjRHTz68~Mtbb=%qb94I7w!4ojL0rr>i9A&$JQr|B|kDA&{tzE)?*_IP=xI$ zD|tGU$E>lFmDb#=$e3`PtRDDu*y)_z*o*x*h{GsF1)BO<{~v7qf4KF3w6SX<+a}FX z(|&~ib(L=fpbquZt^Yq~{eQOg|2*se6UEtM5SLCJl4yyfxCGMl)`a~FI7&Z`c5D0T zEx94JD>t;2_-?>x{`Xhh2jgqhhD+%uoOcQ(^w0J+oDnX(tX|pkP<_(9>u?Xx+W+7g zlwakiy}|!XR(V!2GA>+Arr%LVJfCxZe*r~LXy-r32$$$r(6k})Ke6%9iZ-N_J$+00 zQFB23o6i5F&ef9*i`5M>Dcne2lg4%2#BJO~*H~MEHlO(h zdz`)ii?9Ssu?%G~?LVdcC!+M-|AFrRF!!H~+JChEvD|qpu^MZ!9ve}BCg(Pf)y76@ z|0f@|snkW!P5Wxs&T2Ne|MT7d#qK}ZFx&mlbN?s0|9;o;RBp63v#Imh+_0V8iQU+X z{WyrjNPj0Ql#-j*7$?wm`RN{;u0mWKJ@bzrv2*B=53q&od`CSXkK#DG8bUZhp28WN zLn7PWJ2DwTzd&9>y>;O$WNNT6&DhocqA`uUChj^~`xx_v8^5} zrrGS)`NsLh#(A<$IAx4a8{6Agv0IO^TTilM$@+`zRx%-+BpcU>`{nUdj|NCp&9e+-{a;Jq;WOQG~IHx%FNEk;>Kxv6RWaJc-h_9HgT(kN3 zg{L7d9IfkD%~wvcOxep{QvMarBme92NBe(f_}whb!8|O$A}m4EF7?k?*kWGSindky zcBIjcnp^50>h7q2>Se<*^{-YL(TMu1-YJpJGAzeRBs;Q|KRc|Zw`#{NSW91z_BGmX zhW6V>`yHMcHi|2@U!Z_2LaBPQo!p6%7`uLzc7@%-=`WeLPdB!Y*N$J(j%RDXXSLtR z-tFL{a!$1W=O7vF|2a%XeNiddv;W{_^M8F!g>W2osNZ3(Pqv_N(_s zxPm6_xmmkzLA2MeO?yubbpQLi|EPJ7{}Xlkx_UIw6G%F*agFx>L$?3V*WW zq9XzY#pz~U|T?2{$iybmCL07?_=lCRl!@SDd% zrSe5(jG+qEh@M5+4)*V^Pwn4wHYNKs#y*Xc)okzH(iwyy$i)bZ!WfK0(;DmV8?3*R zt-dGFc8q;}l6_6KceDScS<9BLL;V)^FS|FvPEVo{_3YmX(wT%Q$VYOlb1;qGx{7aM zMNXJOpM~~4(q3r&d#?3wcJLf=#lrK*1t?`7FCv$qgkCY4T|d#ceTA1H?t0ay9}iU{ zm2;r=@09YNb{;B6>;EU~{}HEG<1g*!<$k{stC9YJHbH*A|7Nr2kG3tI{|?Vz+lu_` zH4C*r)ZK{Mn)cVN{gI#TzgaJhXw5F#^ShB=Khv5%S@f*_<`2>sFD>~<4VO0AD(z^m z&tCPR$a%$sbHa9VCw5~mO7GeKd&~Y|dZjjBfrIqJsD4q|)+ysEWhG0+b^YP-P(j8~ zha_5%#!(!{37o>rXU~5VcI{Y z(Odgy|C4)#8T48BeE;7)bEW>;|7Pu9o2fXU{nu(6XSI=w+6GyDQvY<)`yqXlk3W2l z-_OG$q`#jXmXJ-JZ8KV4@odoMo~Dqdx9bmT%q?nnX#ZQZ{}rCUW1sK;b57A0-JeJG z!O~iW zhw=ITzk8m4f9?Mj;}4=Sx)N3N7~%(vQ+s-c1yOpaKpb^Qq6KL*v16O>nG5V-|86lR z`dx}lqdmd?y}|xKgMJ;Y*0GP_DVkNyc6>{!Uc> z$C3PjE3q0|zLFQ#lIzh`!)Hwvpa|Qs6T6ZA&D5}$EI6Re zA@a?Y_0v9IQinHapKG+wh1w_Le1O%ciEMn-vH9!KKu;jKnEy}T(6oa8k1w(Xt^9*+ z;v?UE&pyP_-b-@5gE))|bbQBqcYo>?I-hq9;W~6qjpF_`bo^PrE!=`Mj-u$hUkzJd z_OAa|hTH#bo+0kD{e>s!r*H-><{7PMLkelMB&+iBLyU|ISCgf#Q*g&S{=OBk34^3D0zLbZ-gbYT*E~Kyw4fDjr}No% z`5}$=llh@$aek;Bo*(Ka=X+NmKQxTa4+$g(=ZD6D;!5(vDCv#CI7~oNTq2PlCed53 zN_$^^m_pA-`weL$g*HqRSKN^wW{|T`N}of{L&+`Wx~N>f9kD=o5#r^_g(}4G)9jI< zUOAUIwiL^<`3uxH`e@Yd|H;mwX`!tV2)y+!yC806dFnf zhs4FF!dht;KFVI-H89j>4D4CUt4|GnIRA)gn`H4{W(xai=I%9>8p;h+j z7q*Wd7k+Z+--M0QF2K&_e-nP%e^PjT*o3fS_5|zBqr#8N|7Z0_AMu^6P;}^Dhphw0 zg~GvO!j@fQ!u}WY!!JCCqIZpBf9MsqOQYl4`Qhzt+EB<3yI+|U-dy*$VW;@6r}D$z z*?$*)*8Ej#wtpA)ES?g|59EciZF!-xIWJVi@;sBgP<1Ge4?WNNW?o2tm}MQ;xQ6{W zh{GsF1)3M;g{HZA)?V^L%S(CIUh>Qf^Fq6OlqPF8=Y<++*Nd;)mS^nE3k|FCLZfh! zEFL~J#HCY*BwA29c4{a=nqK+bRAb}RaFl)=)y>YYoyw0nRbMnUoDg>k^&5JHGi3DM z<2mvIqW2&#kyp@qOdo_)vpy)nKk1o9>xkDJyN;Xa`nJ9}8VmW7qW3MkUeA9x{{J~Y z+;+TkXnwd)M*hDCQTU7I{danW%sFAxR`;psds(43y>y}WKet!tORr?(R3L{w0M#Se zVETuc{-G*Qe=nW8806RxqB9v@lCt@r(Ask@HYWUqCKGiM&@VmhTRG`GuDvPOoO$SKYEcjj~$hOF8eP z^DZhM8LchE?->8tahuGOHk$($95bi;E_>~{=dQepKK6Ln42fjhhzg534&=X`559j~v&*#0rUvB&@ zQTJ2Yz#+%bzDHf`?>>9J(O$+_l;0B1L0sAN$)SAJ&BaN<(ZexRc}#dZd{Iv&E^-VLbR{w z^Zn1?>lYqG*ZEGrkomuvr=c%$FaV$1|BF5U`JTya&&G4^dC!BaIb{AHGygAff126< zgyjt}73-l5anXpl6AAQvN0_Egw~(zLvH$O}|B1_SQkB0^_ zF`NBQHcGdCB->m%%di|PkrbC;v#+MNZjm+{cP)KA+P6u2vuluTyB-f4#TD}d6p%$I zrEe#9qJ;0DVzF`^c|7bE-i!FL$Jur4fBB7(W$ga)^Uj;h|BkBB+6Cg;aP|A~M0zs03dhx%r9 z=GW?mHk}l1LG%uA(WZy*|F+6o%W!pTusSAx?dzQ1-x#kCNjvwPI(n2mj?$0V|N5d6 z^h#|#`c~*E`WaM@SGI-9I9FL;%n0YiUBD$=LF9M4Ms~90uaoUx8W?VpUHl!l$-B6Z z2gv+}_JGbG`2P#`L&sO_M-a}z01QIU{W7LS{(&LFx#;@k<6#6j24#akegAj5^?h^m zDq~g5SQ=kret*FCkA68Yq<>_bder=4sQ1wQW&$Q*3YzXj-#D`VLALH-({FbFkw!ae zu6qCXhWCHTdNzLpJ#pGQd}qD?d(w9%q%jRMP=Cbx0%VVW?w0mLpN9qbFW%qu9MU5_ z1J9zg+&dC4_`ZsBqWy|X$l`sOVJW!`@eS(HbM7@?K+H3%SfieaTkhCOtj1cb$3}F0 z;9ikM*pAL8a>7os<6HLolhIz@9)F8_ny#_uQF{BuZ++4`TE_o_^r&nJ*E>va{q)|H z(ksw@$nVtM)J69SWyYHFgnl=r-zBSJ`eEa9-1uBg?)95E>X1YW(m0CaXfmcZqvbvQ zF51rPhmk(1Up~hFZVphp&ir7eu^tW1O^nnpn-esmSU-J2I;U_3=TN%G`*-^53-rnq zA9~`EaEX2e)yD6*c|+`gF&x*#UB^w_#$9+nG??~qy%t6yvlK|13w0h18<<`Zk|d7ww{l(Zn9J`L@Y)xGhqu|+?C zlCjor#Lq(1?&py6P@HWZM8@Q=3O#;5{lFq|quh_mlj=V6 zIF1uIg)=DQ$1g_(Dp7?P;;2TAexp{OQAgJ6KN{$t?LTVNw=}WwnlpE)%uRuZ1oIX zZ?n-Bdtc{|?#B;@nfnY2x1E0%_wfLk^OWD(TyL_|z3fZoU;qYT2y!t3qtIoZU-+^0 z|L?0K!kypX^CTx=5~d(pGtDQbVFqR)dUv8k{2cmc?{Cker^|+h|N8yyvhm(OM+GWT zwb;7!bZg?Xty7cNwQ{rhb1-}m$WJ=rkK+PeJ%NqS?qxaQojM0!iH49k)9+l008mGoBY<$u8weDc6?Vuv2(9;s=z=IaOqAkM;M_%8!cq z&RZdVjra}9PgYyM-|IK~aS(@5iVDP0ho+B||BgI%D?izGL*86Bjds)wQ~s;UFTS2^ za9)B;3OADV7wOVzK^jN#+4mC)zokzZE^XyZf63adW9`D}$*wWpwaCxvU-2G$5!Anu zL~c0lyi+Kpul4-S&?}XtLVjZFlyTL=vc<_BKMwzV{budJc?t3Yc?n7SYu1~t&|B51 zmK*B*RrUTJ-_>dDfPT%f>nM~)`gL;vW8qEV&R`G0pPvr5=^ayZL)RbdA^6VI;jZv~ zJV56E?z=|W=Ahb9C0T)3Kl4(udZKwM3cr^fdONQ#axegcFa)`1{zx6{(6;Ys<77&> zjcnH!q{-UR_Wz7D_s%m1AFf@S|2CNWHlAmr^!MJOG)7?zino|wkR@a(CeWuK9~IKA zysH1{*8fO1j_R}Q^_j++`NnJ%>l4yH&*G0UFTgavE4rK=wxZ|#LVb5tfAfHB<4Zq% zGC9LJvoHr;e+Xe7xd4lz@N52$KtI{>u{I*yIW#vcB_sQPSrq=V{!7`meCJwNF1!+} zk<_mzu$JDMcsMs*Pv3~hCQWZPevr|-;*-P`pa|Qs6T7h&`%#u*UpFe&*yqjmt}+(K zjLGqG^8?fv>ucFyb@z<#cZ~1*jPFQpGrn(Thmp-_Sz&(mk~zl)a}P2t+)hSoE1UM& zbLm>q9@WESDJl?0SwH*3H@JVZ?GZa=V)jwc*X zl8x(pBV{H#5G@P&=g~$_;j`}(MdNRs>lA&-x9^@YF4L0-q=6QC>o!ulY5GyLchk}H z{evCWCBzjEHisckq4d1<2l55hI)fcrrzqy9%xQW}ii~D$hChNb=Xkm-DqK%$H`l2?Ve%6$m|DsNs_0nu` zuM=DLYZkw^W@}#jHvmBd)Xm5NRu_M~rd)^Hyv3E!O0xZH3 zlpf&U62Fwb49l?+tFab;Ie%F%yfKPHp)t4UxAOmGeoEn;Nb~dWCih}LI`nr3${!5&Z&=tE95m?$4%Tu`g@Os`{V;;&iAa)7hO+UzgevPp!g~ukv0{5_bJD* z0Vv(ab}V}=6!9Tc>IW)P`kn*&AXKl>2ce8@U5*MiceGAkW!*hCm;H}w6uIVI43gdu z;GuD$F@h3U!d`<_4jqI zkBS|S8>iXj2iWE8`#8P2`SCE$H6~yZrXU~FFaxvD#0Sv4kB{IG{{!0WDM;}%q-XL! zpoWcK%hs>sqo^kvgcG&=4`@X6{{0;3%)dTW1W>%&JuFMNm1w?^57 zQ_g843$(SK_wTi(=>K<0r#o)kUgrD~_pf5M`#Rr!v^E*-ABqcCyC+p-%(=^*w-T$d z7VEJQgWU51vdlHgQ88FLsG`SS^n4KcCqDNts8jWK*cYCE!gKv>f60H-KXpsHM>~pK zV>@s<4O znVI1P{SZ+^0W4 z&-%Z#{tN$~aD0n7$T0t@vOs%7U*uo_24M&ao3cZbd)>T28z)=)Xw&-gp6|2K+cP{L z{XnfgqK>S0Uc)P%&tlJKKKpNWM#z=U2#mrQjKc(UeT7Z&>zoj+{dau*ao?A;N9gUG zFv+okXx!YO-D8SyKGMH2{!aJ4EcH@3VsO|! z{;9BQ@>AiBenZ1gUmP0FxsUbi=Q+`}jg2$Nc}NQLNz8u9Pv%^ZVge)Cve;R!))?*_IP=xK+iQU+X{WyrjC@#+qhfimP zL+7)?J1^O{Hhft4@kHM{bF2bId!7ngFFt8~YjD_t?fnLZpVU4X;*Ql}r?s@7`VW-X zPd?$lnfr&`FZK&>uIcAHl*VAk_a5uxUBs;L){e)lZ}ssGyZ8T4^zJ`}t&@s=#?}{`291fpn&kyVlzf5$8U-@WfX~rko{wJYq&pq?wJAAZvLag={ zU-=)^%|G(d-U_uHH$qLu$NJU}LtXdB>efem3?GG~(mak6IE6DfhYPrbE4YTT9a*7# zPgbZvWl2`3N@azZdlSEx6{>G#nScK|TzB40+{Rtp#{*>kANt>Q+1~%i_Wnn9XeHaM zIi|?8aQhA4Cf}D8YEd_nuL=$J`X`X2H==l0R!G19=g?bPeUXFGg;}9wVpbSHuUsr| z7(^d}>I2@fK@3&M6<5!nK7t&Br2b|cIRT00hkZ}${K)^^9k%}Z&mqmGXlDzwX(K7J z?4t6|G`3qWt6X8u!S5K`V-8}DQQd5Qzr*^E_EMm|Op@jlfTAe;u6IYs zvf17IE8U^uba$vk)va#pV%;H*YK)40(_g(Etlpmfr?69aHwu5CjA&Z(K>znZ|M$T2 zf8hB)@cbXJ{~v_*I}bw5u5R=5?ofw%G|YdXjy}-;J<$I>@J@Jl*ekuFD&r#?az8z} zsyigsb%%rW*7@C`1&8USXx}I8U9K^`+cT7JamB;rm#mAK={k8c(hoVuZ*S9EM@k!a z>G#pzU)p0`18w6U^z2J39`;ZEjN#!kDs`O)&ddCk^51w6DqegLDpB}d{fzbE_$$ga zUpdz(C%v!pI=}KDVEkC-=n?40O3Ixf?O2-jrIS($qFOrT`xQcqsTFczTGp9 zOh2z3e=R#qpeMD*1V7m%daE(IC1s6@o{#p0>W#LRBHN7Z<-@eWf$aa057()py**Xh zO||hWt}Um(Ywg$XW?&ZPU>+7=5t{F*gSWK3+1enPdQQ61j=nvTrq}k-2DS5g?XWIG zzRqidr|k_Aj@IWRf7}x3bpFx%E95f#rTws6cqLY2JvO2MUC)?1k;NaeCCJa-|JzCL zu}}8-#yCBCXR$(mu$R6cJ^nZ4>M7?g-zXG!7^SE{9Cb*d%RI91oh)mF{}ft;I}1HS z@+gku1nP|$r^qeu4-RL@bBNZK6ZhB_^wtt{6X#r_UqSl;X&@!OEoT02&9%HAJGKAA zb=Ru#k^h^0J=*v?(zopVrrz3zm;qgPR)Hd zwnyu-HH+Ej!`axA+2_)27|gat@*?~EtoH}@`R&b?0?BLb-Q-@B()W`GQF%!Fj;Z%0+P87JTHBAl zq47WT|FK^WM`@t~anvD+7X1J9|Mk58b5`E)`TIZS?mg>2W8Ej$IEoWEg)=yZ3%G

_DC_gMIjH^NgW2b(qQ?*~XP=|yJp1|v z`}!VR`VRYg5Br+!olLK^KTUfl1W-+vwE9d*|%-RAvwK7?`f z%6)9#UEUF(PeOG!+xHe5_#Rt$b558dF1l{fhrUzXrcalbd~wr|xXLfEj$dH5y$iy# zkeLpzqx!fsC${>BpMODlTs(0^!JUI z#_%LP+20)Xj&YmbnrB>{=otMd+SkyL8fcuwadE{5*!Scql+L#gjy#8wTJ36;dG$KB zz3?T($D6yNYOc8~$|hUi!++QR={MJK9XD|scX1yN@Za@+s`u^qANfBs|DACKeUXE% z=lwSZS@d&$`w#dLyqgn!d!pkL@0*GL{Cmb@L&z15-l-fxj=~s>L*#pj`rrxlR_#36 zJ1~hp1?|JMKYaE+N4~g@Z+O>*?3}8elCv-e^RNJm(8({kgj|YcDEyOe6?~vBmh#A<_VkA5(~tju zF#rDw{(s|Um2oph#!=0FuNlk#e~ka%{NP`_U*{T4jy0PU5SV%zlpweBm9`^^cQ!Wo>y1zf@vG?{ld^Sic? zt^6@L4hA48E`dSx*3s(UNOhNRX;rIVnN^3RNqF#Tso=jZi zKQkZLNH0JUT9hSvm$}XOoto^vxv%Xg-QoQk$9JM+i@6UOt;z2t_anYaJwnwA_3Fi_ zeVS+Bpks$oijG2aJhJm|%P{AOslAbbf|a1GaS6J^$j%d@?I?>+iTvMR&-_kFy7KiT{D zIOm+(xQqLEfXv1EO!P(5d)~i4>;3!F-oN+l{kD6)zkkE`_xJh!K59mK|9-6Z@1<4m zeftJ_!nKmF)o4FK{g$Ugj&ufK5QZSR%{j=WM`^ZnJRL^RN1^>aX^TsVYjd43;);(v z9mbIpPbqw`NGo>r&sS#&T{2+tjzj-IV$d1|3?))hIpRm=2`#c z{@=pyWQG~in1!wUD@AYn{@rQwzw_Eyhxs-B#s1&f+W&mdbFubM_Pqan%lr>L`+u#$ z&yn6dEWjcpU$IBvMQgtF)?L!xW-d=(hMxVu!l})!MJ|uxJ(FU40ansWM%ybvu0^T6 z0_(|*sFcTwBkJIL>LOXaK_5pJI97!1*oocPi~Tr=!ze`s;;2Il(h=qi9UoYuw&vgY z;nN{{?>_QvZbq~&a@0AWz5jQdegda(23=oNx5x{)gey^;bgq%taTB)@?f<_^-p2!E zF3~=sJv3yeb;x@CL|=Li24E19;-YT=4Waj}KaTXBVfqNPo3o_x+4^JIL^jA~^A!BI z_Wy3`{^$MQWBk5|FMVrPb{I!5{C-wwn(6tYWufPfHhK!_b@~VN?Eft{$4bd-!rlXR zPXbBDqW!-Uq%#RqkdJAYfm!JKZEnx|4;|lDM}#}=J)cJ|z#=R`wEuT0xeUv(5})n= zT}`hXt89ELYw7DzJ&@gn*l>2(XucP=Z#mnyVm|x#CHC(Iwk6r~p4^MZ1i#sc0u-TV z{%YPBegAg5aF6}V25!0csr}FHO&$0P`?rJrk2=(MvwzvV3AS(ao&3i0;x5{|E}h-j zi~Wd>CGPYN2kEUNrHvxP|A=0VwBH=Xah$*@oWVIy&G@-{`M;i;vG#E=1p6h+&g;_Ra6v zyLQo9`$^y4bWZdQpHg}by~qRK`j+weSY#g?pS7LXFyk{>E!^?;7eeP#d<0L@zruI$ zly7qS-5?A>=Wl)cgY58r!3c5`x}I^~2P4B6dh{*s*xZM{zUr6M#|-VUk2<^1_2!#X zeB3XLbKV3@Li&?_VG5a#Y3TVM0rQ3Qi+p})y1^gMuK4VGW6^)1CeE@8m9LSkVK4Xi zO`ex|&SrC$0_+vHZ^hH$=f|E7zj*Jj!~S(c!htnI!rRCGI{d>u z^Jn#F*DHg<8@2yAyt(2FVfV=|ggvMKD!lc|0N628V)*k7(d-IFznoC?aRAlukRTa-kken*ggHpQ0iC( ze)ghoV6J&I{C|YqeQ?#+dFcDKiEskP;FuO#sF4E`b7F2x11E3_4RXvBrZ9miOke_2 zn3x;5H8*erH*f>j$e;?jP(unaBoIRbvUa5{ZN2X;X-nD)OkipzW)4hX3KN(J6((>3 z3QS-M6O{W|+lDx4@97`kd7icQ@3;2vZM{6}Sugu-()ad*k$hkLiRodX-fzvwB8NQQ zxnm#4wZp;*dS#+09NXcWJy;ap-CGp?Y4o3kBtKHkkUtC6FaDXlX&zR>ysO0LLfzOe zgoYPC$4B!yWy|OJzdkP?eTM(*^ZIwqsXP0ba7vn|aR%pb0hh2XwzXq>@}fRb;p%7g zZ|K)Z88cA3SpSB;g7kX*8yod+tbHb2b<9nCu>XiPr|5Ta9}m&akKeJ-7y)#mYn8D9 zYx&K|p0UP1jMYD4{y{TZ(0Wq;$ou+7$hOVKM4W#n3`l3tSGB1y6xlnD@%wOk=U{yp z`dCKLN1;dGOFp6RVx&F{j1gD4_nA;kjz=={Oqf7UMAdfpb=ZBqVjP9=6x7dlU+%GP zq5DMjcI!VPg<8}hjrt~gRUEY@rgLVX1hX&)WgVWOw)Hl&PqqH$Onn8)i_ZPN@7=!d zjPF~0*F!e0mEU~l&9l{i>Y-N0X2z=jMjC@RjE`D+3(%4n78a2iav52!%*mRcnEz~1 zSRvf0evadCyC3)u?dQTcf;;Lyzpcg^RL&k2){#|Ya{jQefnM=5_4T_&A$3i8a#Edl zS$X5y3j6!G-fGuWGgW!xx_-O=h~L%Gv5#_-G`64=W%w~Wyn@_?J!p3=9ckBA<^Hwt zyEdx-wXO1Gk8q=FY;w)b$K5|#o7}(u&w4g(HS7gx^py_}2gu%E8FxV*!7(IJ{wZVM z$a*v*i~e8u-pD*!?pb4yEchSZDqAFED?VS7jOy5z2=m@FgRSH*LM4Vd%;|#Uq{0O{?~i_ulxC5u}wL%9cAq7mfhAe zcT5~R7xxglO}~r#$jxW}&t|hPXS=_`hDQ$``fUK$V;*=_9>7ox#|RYKC+f)5Yw9u7 zsm}`SlUR@ILukDA;rK%`?gP+zL*D_iON~F&Khe(4??4Wnh+_}BT|*pu5d9OC$Mu2u zpD`%LcqH{dRGqc{550DX|MveA>66f)&aFopb?WGku75Vg@A>bs*)fei0~O}PzxgBM z_vy1R2lKE1i%_k;u376p+|Pg1|FFa{%di3~u^MZz4ja(ESo?Fi_9xl-9{cwq`xkk7 z&vEu|6Z@ls{jrn%f!6i<29VvT{kc{gAN?ljY(Xi?kX8R?P(gqE{p_OeLCn*6#5$}C zc{WiTQ?bu)2auH450OVuMNdtZuY7aIgv)=X-5@{2b@l5;%ZJGSopRl8kN5M8b5hgV z>-8^bUyz0R?ve7^y;N$e)W>wmB%0|JU-f*ytgN8tkVl{Xx)WsYyZUd){=b$N$TK*H z3%GPi1!%%WKT3*sd`HPuh1bqyO zkrkK0czWl1@}GP^fj$vE!g=9tvg@pV#kl^@4~^|H|1Ylp^JF~JS!?w>ehQ``|LzyU z46+3M&+!%T%gmw|?!Rbem_u(_uMIN!2?_LQSSb z9xqWomM9-fLYk~UUJ@D}m4s!^U4fNYjWt+@4cLVCg)giBURM9Ttp0mh`S-H&?`7rR z%gVo(L*q{K|IvJ~#QF&4|I;(bZhkqmIVS13R!%PoTckA#$yZ83DOrZ9g|2I^>q3R_ zF4WT-P=_=M{;!3;547xkIqY#vrsL(1o&9pyN3ZxIyP-&XP`Gos>znWTX1cx?UEjEu zL-l<3zu5nvc9nFuN_VIG@9;l3AdN#ff@4S`g?coj{Z;u7Idq~6-N>T{jqkhv^X?yU zt%KI1?!U(Udlqe|^t`ju$svyuNDe8n{$fcuMX!Cs@Q2B%Bj> z0he$ES8*LTaT`zis|Sa0S2(YoTaJhH0sM~tOP;HnD*wrY^<;zap#R-BkD(Zj5g3Ir zC`S7Y?QP_CYTu$u+qWBedJh^G>wjA=uaYe{_}(9Ak0VQO>lL?LK9NqvXVlX_wjMvd zuzpwf$HGKYEG3dHvFTykq>+CyiBe%m!@27L=k471)J6Xx}+AbRc)o8U}lnhm) zt~2KQ%4^EaSF}ly^dDCodlgmp)T$f6B# z@4%;|GZe!y0@(+Sxns>Adgo>Lk1_OO^sx2w=w8hJ$9QpZ52Xp@M8vg|Cy`SS_fU#s zCu%WGcn0bd?hCQ)l15?wo3rX^)V}oZ^M8qRW+AS9Fppe-MOcDm_?`T}L!LY=Z*G@I z(UbYl^Z&NB`X8#~pS1FOvH7#o%rq$k|nD^KCeaAOm z3+u=YNIv3+z$SX_MP_^BEmeiZWE7zfs@BkBsS|??K<| z`Zm}iz3t}VSdVR=xC1zZBRGa6%FQFle`Q!G^p8z-KQsAg#n+=5h5i5jT0a0ij}thB z(>Q~3xPVKzf~)9f@0ZsNGylhTDBP!gaFe`^ySR@y=KCQzfI+L2)re~g$2I1L(hL5_ zapnZjN1*4PwjsJ-@U6b2{(s5%@~M2>Ge0u^e+R!J8aj;Am+mO%jX^QCjh4?4eGB7- z+n4h1qu^)X%qD$}ZHoM8es(lg8UJ5n{J*qXnvDNn&$dNepT7-NeCv5*zb8s7$zGm7 zPC{jx&!3EY5yU+S8q#cX%yB)_#Mg>TMf`J?F_F;;)9-}hi2%K4D;sQ99NVgA)v193-i3`wL=kLsQ5-@Wqxe)+#c zyZpR%`APOK8t$=M9g{^4`N-?m)g({hG}`5(4*4jD!u}g%_j~f^9r+WDyV<|9)&KL= z^}>64?Sd=x&h65kYmZy{b@T}55BiV2AG9@YimO~| z&LMdh$pn8X`4CmghSV_kb@F3jKzPt<_k39yAY4bL&wtGRI{No-@q@@O{q5E-a7-_I zt^9j@8b7hFv2gwuMPUTl|A`Pr6@J(6LiT;lybg8#n}1^*vT#ecwhWm`v+t|4`RNle z3AtC)59pFNyXSj1WX}Wr|9+c>87RRl%)vbL|IBrKy(skl#CIs%_ndVu$t75Z6{wu7 z4}>iLv~j@X8YEvd|99M=u#WzCp7#d&78K_FyvGl8oF9m+zM=j{+~c(tb@Vji7_+!m z?^Dhxd;f#60&&d1CgD<4{P73l1KQ8(|Gub>>Q(=vYp=TLusUkL@ozhgf19iR!|&|B zdq@3;HZ=CK-_bmT{f^er?0018Z78fSFpT{!t`lA2y772@-!j*7z`Ly=cVQ3q#W);7 z^&R6E*z+lNe(hy`|MxyJfAOgQe375QamSEE3fblA-*MUi^v>Dt4b3qgJ>BjN-H3ht zIdPS1#gQkF9IZV=o<`NG5Bd&j6V@pdK8N}To-68*MzwQmn)v4C8h_xMs9WyckoCTi z2J(V)J~}?&lJFH=MgGf)(7v7jdpH004*u_h?Ejt0_HJc-hW)Qi+BlQ_-)sC2TF{Ej z4gPQTWg9vN%U>7S`sM+?`B`~(nEWN~@%Wy&XUBD%SRcRW z9v^uI;vbJM$)4oe(!3a4=f$)(o+UsxQ@(c_-< zsk!Ef8xtPKTr?am4si~B8g;Xa!v)8*40%3WBCjAzze-+5hMpU2T)?pB!%gAa$P4#~ z?;6R6FTU}b@d3At45yH|I&Z8o0Mh7@)?I1d z$93P{LvjFv{zf|pLs4y9LJbPz0J?u~e}Ti#hsycIVYuT)U=+q6DXt2|^ji0xdZjpw zr%ynG_=9NJ?)Xc>q;Yts%DdzlOivJuy^Nt4Dob6$dAUH{bu$Q2y+s zu!Fv9+=%e^FOCR@9Df9R_KphQ9y=<$_3VhS``J-p-}*7(JJ-g9V}9#@ds8@Yr#O7q zx(aX49TWDCE)IuW+xL}WDZkgF8NHtx5#G6@e*e^{@Q>r34@Zn2IehR>!aFbhiGAVt z(YpUA9Gw0~;d@Pg6u!Ij55j?(KM4Er_Qpx!JJbKv{I@?1dz=0={KM4$EqpsYA@qIg z`EZV3p!ct>$uEy({l~UD+7-xGtGk5rIDz>7`#<}9I7L5=-gk>bAE!d!>-JaZGL~Q5 zIb1;bJ8WPw|8+Jkc@@{u|4IFSMaAJJ{Wk96J|1EKgI@FfV<@)GR6Y#RK77v@dEv?j z{FvHhM<;(Q{NR20Z|?fglz2Tf4|zSbjN2Mo6W3i_q?5!5=j1=5o%1yN`7Qe^ z3XefC#$y7GDo@P~4V7C%csK50H`1KxdryadN_-+zkF(z7$xrA*|7NHi@=bkK-wf%o z8$n5q3iHV99Y{v=GnG|WH=W?>H6A8CK=RuAvc z{z$8nJJijHdldInXU@!QM^V@!T#EXw z#!W8N4moL_-A3aNU;AL+ywqmxlZ)B~_p}S#Q(^tVYs&Zcygx*HtIYWo-{4PHj_jiE zLHkzsKiK^bbN^)5SocrHF#7nVHh%1T zEUf==_=7%(f}J3)X)zxpV&6hDV&7w{@4xK^8-#3`X>D@p+{RrzMC^OWbg&udvCkpS z?W_1Z{Rcl777G5e$@=ei_#X^f6U#SiN|8g6yl1UVas*-@MC^O0#VFx1DC{e+)jb~9 zKEgKbw(Z()Wn};V(AS_{(D$pN(EEwF?nhDR=NIVzRFQS7o({#%ACKPhM3_MK{ghvd zoP_cp4ht>su_x#m^+%ioFpWL~CCI5uI@9X^4)s6s^qwmADf_=^Hv9Ay_9&(_5wU&Loip|QNPm}d%Sj_(2 z!M1(LGZ+~P<6oCKCn>IKzw!6<+FtgrG59O#tI=@7x#H6Iq=hx&T6P%!|2N8~=e$$; zI&mA2*~|9dZT!En{+ooiAU|2Z1Rjt7uVzOV)<2l*`esXWxwK#LA86Rf26s*wDzFQC zunz}t2<=BbALNF4zJopAv7T?j^F?9)_b|`*J7sZSs7Pt?_)FS9yGRI3k^6 zNFo)bxGKli(`%idis#d_h}RVNW=tnstKV|s@;HH0D6~hW`ks{gr-jep9Ljw&c~pE( zJLKO&xFqfhs*h`fpOyDet6g8%|7xl}GWnu`yy}?ixQW}ii~D$p0kr$JJLJEZ_d3zF z_XFF!u>U5#k)7YPnEk(8ULy6kN!{hQO^Dt$^H$ZAH5j$!VL@AA7uKPvCZNtwK*n`6Ola2mq1QI)wBEt zWZAe6zdzq#e~~uY+r~PJFG1hG*jI>*!<^9rF6q;Xn-k)zEN=Jc7J3r2lin!ZCUh<)0U>Gd^Xy`T>Rh z=^FKey7S5Y^hd3qb3^@t+X+l{7xc`u@1C?iYM-Wr>(Pu0zK?44MGaDO z^+Tg>srn6%$DviH)qffGdAItHtP@UOHlAO5rQx`K0O{mVp^q@`DVC?7z$u(Y`&$0} z!P-~}{R8UeF7X#d|QAEN5j-`f8J!h^o<**s$Z_kL*q z-_!noLHmEI_pDuBOV+*Qz3%<({GX?sGZe!y0;5pYt+~Z_|hUoc%TE760}+e;nU-o%|v^1=BDCC76Xd zh~rP@kqfZTx3`GQ%71bE$r5_!R{49Q{jcdO&@ml0Pr-b`H_pxxAJcDz%fR@+Hvn8+KDy}2@DjQ4uO?qMe z#W>@}=y&mC{*U(_``_=2dx!zVa$wL}-wl~eD}bGH$M>=m%8T9zeV^8T_{mdYc#Qj8Q5Zq?zin)eZz1lH7RRcL5;q3L@i%^j_T8nS zV}GeOPHE_@D%Gb|s(dO{K9wq;N<-tS($KWFG&FB64K3?SL+jE~ee0zm>$tW|X{cUY z8fsoGHHM=!)S?b))T068{m%jSHX%ZP3KKC2Q&26<8l+H*y1j4s{@>95{)YbdH^MZ> z%s>fdVGibD0T!Ws#~a%JZ)pF&q5u62{qJw+|9V6J*BhZ{`5V6fH}t>1q5u62{qJw+ ze}6;&`y2Y--_ZXq?!g;jiFB4>1y&;ata}_?8dlRgr+PM6LtlrU!JZAe9nG|L=@rCWj;TpZ9&UYDF`PJv0cUk>THYx|2(7f5RUhnxY_56{Y z?)gs@_mVYxq%#NeumF$84=tj{HhT`s=qvF1^Y=fx|8-mwe5GUhfAL1>@ACc9dq1gM z_zC`{ls)lASQF#ER2tTieV;B3*FD1}^jfxJ3WafPWP^G!?iEv^%zX1R%IO`}+i_eK zxry9@Bt7omQ%WyG1$N=_{ylr>o!#0F*hfEr=o83qSAUXS`_+Fh>iax~d!YVD7rh(#lx<{;qH0J#CZY+9+h*dB^^?P5c1y~7TVX!|AXaG<-imBPdG2! zLpEOI|3tInT2^`fTRnd=i?+qq|2Gz^_pjLet+5aA;~$Lw8^V55{vGij$B;rjnsHlw zkR|)c{;$cOU-kT~?^mfz&WW$utPMw=KytlumpqL)mm-Dq18s;qzO5VD5MQa1y@nMn0>sI4SJM)+|A}pvpx5-K_9S>@39~DvTxCh7CftQL)_%}8)Iam> zQ%DX)a{cfyoE(9wS6ugG*NyUbwG~nStm{VIIQN0-D);A}Qe6~a{&#pj-u;vH-@NyXaN+$s zuH~iSVZ3xEU?L_V`-=N_k5lNK%7C2kH2Mtm2(`th&S6)?_^zGTQCVS%H;UjWt+@4cLTs zZJz(t`VNinv;V8u|Lpk|w9+$a`CA*m4TbhPA68srYK!#xeH9-@P<`IJ z&0Z(s_Fy00{AcyO@w%M}_Qh+~*AaIJJwv=_aZmi0tE{mj{uq)-AuBGk(LK_eV;DJ9 z?v!`Ps)Ob~(NECJ`9)8WsU6CPTgrzAF%Kyp=ndD5EB4zN8$z*4QOoAf6$@-U@lu4txMV0v)R{(`|Cd0|7W`T8HN3SUQ}<6 zRzD-I|G&WhEW#2jLvp?KZx-8Ej$W%B5a%YYr00L&n^Z5?cYol+uOqiD)b3vD`|%yN zsPA{^0Lg6i{~L{ma85;dPWk8=yyX91@c%Ql56C=XUw-vo_lFc}QRiOLs8^3SV2w1^ zVFNbd*f@3O)I|7!zU`yW7KJDCKeef!?Ej)1c-+6!SmpU>2Q-s~^@-_)`4OIJoc|fu z4*vGx&0&kQOHqcnhfmdHWjj62d5CiuchUEtVX12q7x!q4a~O~8{6zT2r<@cr(m`L3+t*fA^|IJ!Q3clP@5_Uw&ef7AM~@90KrIBpF8aBXwg`~K#zYx#!o z_p>+fqj|pDo(p@FwclPfJiOJhA?&7?P2Up!76<%(2o>u$g*Ojw3gtt#gdJ-)g+uyZ zzOP<6;)2VKlQ(an%$#9 z^^Q@YHt|Vi^{9|uKO)p+MudirNufUR*>H5Lyyo1>qs8G^4Zm#DnDFks=fgi;`;(BQ zm;acJ{-x(avtzQzA&(O{g(maD8ZS-?Eqsj4HztP6y$PW;@##?hjj`dh-wX45*>JHB zE8hPZap&-6JKyCO#)eDuE4Yg5xQW}ii~D$p0Sww8zoKuO`N(AN4~)4Z`;8qOL5{*0 z6k|LlU?L_V?(a8+?9=~0Er#{c&WPbU{vYjt*f$=pum3r|WAU?)Z-3q%!1DhM{V&3C zj(^23o(l`;S^4%{3R-viA9~iQIzhs^>!~S%!@G9RF%3Duj0- z{{p`~pMKY5etc9P<)23iwWvcH^=QBz=j_7)9KsPCLlP;pr;QKD_#eLgPO?k5d$I8W z3;oZ0{!M8#p&2b`tunr##`prh|F-@7|1VlMU)wZ}Y4ov#`k3}O-kV0Yp z->=o3Yu*2=?ti2EN7gYd2a7|F%p_j~+Dc^!(5~?zzxn9%$>> z=ajF{DPNxpZEK$k)vtPgNTC*W7rj5!zvulOe?HuHjdyV$4>5p2|F^t?p%{)4h&F!Z z!m(i#y=wE=kla5gjG-4}Jf6(YzoY(rq)fP{{`Kuo^jq(*)Nf?p>xsf%9NRa_FI$c6 z5m#7$;N7AyjXnbSGuj?M7#!v~b^#V4-&GWr zkjtpvsZ@-)WmcKV8Tptl`P#@QkY4-oN zoyG>D?4bHz{n_l8$NqyXJ%@ah^VPrF1iwH3ulg1B@B9zy_r2S*0GYfMt583IypUKwop?A(!e_#=P33?7| zBcmH#*zvkKOzN6tep`X$R{iv-$DZLC8*o8gVhXXi-BRGa6Qt11tXGg~Rx0#H6fmt$#JWk*gPU8%!Uz7il z+Uz<}M^EE%{?FGv9nLxK0xsc6|74B+N#XW|p8rzM|5f=PU8CiHJhA`(qx`>K{zn`7 zeiZ$6eEr|jci&~dX=#@Ku|9`(cKsLP{O4!FO>wtz7jf**eez+{ziEw3a?nQQDdPC! zq2%N7$HVC(FbeSFyZ*qRsR`aDd4hz@mV;ozI@u*yDKKOp~!4I1c zPBy$~e7^XJND0^OHs;`(b^3&-U>as1D=u?ycqpNF9%a8@WS`RKpr?xc&d%;`V!xBs zBiY|Cuv6K=wPf8)_BUBC+&~uW!w=g(^PIl`i%`xNvxIDSO&!Qh*S~-+dN=Zq`1$Xd zf3TeWEuCiR#k~ewrJotc|4wfk&HrBYOjstJiqBYA?@QLtq$kByWu6Hu>9xJ)&>wsz ztfsF)!!`2|9+*3S#~gZcow$}8_9`SdA-mTe?PMu3*IbwT>~x=H!WDQtFR<&N>qNEd zsBvwD{THUYf7hN~?EYPM93QdEIeV}V2XF{Sa12Sb-}C*+-#K)m3vq9kxW;1-p6tIc z#QmeSSN)GH+SE^#X=5{_Q;%k3kvwnxeRf2SUi--J_k4%+6KHrr-az`YZv)$w%fHIN zvQ_M_g>3J4*w)6wWJmLrIlkQdoik)%|1EubUE;fuzsT2*>K)eq+Rw*_T73g`{C#m; ze*Iat9t!sF=NJlUg6Y|Q9?kY@9C{SRne z==uWP=%2OlAQ5b__ zB$cVL>>E#ybD!fJ=n3?RXgKfrA?=tt^!e_3gFb*yu>Y~`(TDYkcK;;DPr(e7U>5rA z%`k_Y7j^Rw;+TU4^uqlArw4~c^sKnp$Fzjrd5dot%jhf6yy2wKP!@ex+ zyV|W@XQvkS*C885e>_Zc&PuGt8mz+xY{C|_v$H$Yx4B{J-@)qNvFcwkkDjUe-_@~A zXl9eQIJT9&m@)n>i#Alg#D5^2GE`s}lH%gNc6;b?&!H6d(GQ?uKl{Ii{jVROj{Io( zaLDgRa17;N_Kp2De;hr9il3{8zvR23HzSK2^5`on3Ma_k*1_Qvc^YSM4i|6ANQIf zpZGr(vhnA7{@O;{e8Z317sH)D0_C5ThB1I+=e`!9MUqUaeKc#LdtUokF{{x!k z^_G+Bf9=u?8TW{4BP)LJg%Hnp|wIA#iBn{yi3a@6=4vIP0r^6q^3a-)1t=JdUFuGQb~w>g-H1vn*-Eh3j- z8CGB=s?}39i$BsnS9hi5>G~$+gMP$DWku6v`I~IHC4b+OzaPopWSeliGNv#-$+4Zv zm%{$TWFB$<;kbYHYX7qa>#zZZ{Cmxq1N!6qyMN$Rnuuis$}UGs7`@5-F6I+w+`kMi%{i1NpCiA@r)} zbHaUJ6DCjK6i%b%tTDsnIb6UcWcPZ<>h>%2&TiiZuF|igXQMRmc>l=i?wO&+eB#vT zk`U*b*G(=7>4hbs9u4!&yLZk<$M@V6zKy%Mk9PCgJ1&=m9N9VdW#9D6`uEMRr}w<_ zGXIH%;%q2Z}w#541C1KFFV*TUV@DzP0 z8m787#5D@)FkD>A;g`b*aul*vFNZN?F*50wL+-VgL#K5H#tTnC-nI2u3!rPQ>sapL1|_0nxX!T(<#!WwC#}ehQ=fBbOcU%kEy43TZ?fHARZDjQh`D?%Yh1x3Lv^He= zyl?!h{EagIQGs3P`SkU~9&#TJ;1Jpu%U?_7>$URND*0=xd`{+td&tHN`&XLH+u6Tp zT`zwki#Ejhe@CQq3`xX&fU=9(A5)F*^DT60ljqjEKEKr?&MA(2Ydl`RrdeE^|Cc3m zNXnae@&u~ldD5w!sSjWHH0tjtH&ExeG^%UdKT;izLmfSh`sv;kwkancm%nG6e-3&3 z{#+ujp#6sO7ddUN&K=71!}8X4?;1TB?XS7)uf=SU<;q|3_t$^y|3iDcNvAOX;irSc zb$a<%)GuWJk4nOAvX{I|_I=J81 zUUglUUDtWndC_$$_ec0`6e^AHzTo^Z^kR%h(zxy_b-@IB+>1Rm%s6BEBs65yP3n`l zS9={fMcg#ZKnXtT-<@Hy^eR^|#E0llik32m&Rr|;DJwcCsA}KzC zQ}ok_YZt~f3<~}K8;vy(*P?tsM_xc&SMm~h1sQr_T_E4yRpIN%dp12N%tIxs<&m10 zPf&}xjPGB5Ev%1~*8la3r^8L>-Ns$q$3qNY&=$|1@2vwlez#6MY2VYMFRt;t{C`>g zCtF@qKX3lfKA^X~YTUduhGIA>?`cbuRg?MOFp53~#fW1NYDc<0ZH{!)hxI@Cc-&QC zKDy&7e#vhBsxsn}Plt)Zt9{#($SIhH87RRl%)x(F{>^jD0xZH3EW-+{#A>u}l>fJS z&Sa;uqDvl&a}e|I$^XaY|IF`|e~Z=4-pAwdS8<<(a%&UBwfxseul##YhjruzY{C|l zq6`&?eVg0R|0#2p$z9_1U?1{7d^+rCwO`mL?RTc_+ZRTKUj0yq{MPps^PBB2c!Yio z72y-%O??b4hlhm;!bzl1k1YEylNlD8>75VQfp_fjM$e&VG~1eAx!W;a$cwAIXbnH| z6q0*~h1292R5>pto!V}Dya``GJ)5?HZCmHKG^*(}ud;tpi#j~n-;RFCIahEM*KrfK zaToW|KHvHS3yt$fC%V=fUw}M%(0Kg!*54D(Ad5D%+;}EDlumwIA`FlRknMGh_(A`# z`yb4P$58rH=%MG)t>2;xh5dJ5@jm{>J_dY|6~8pT{wI7tjvawfDE|f@Em@4-pYivT zeShh>la;SM6DE?AFa^_)d{zI0_!;!t9i9zJ=(Es3k83l;F&K52Bd%pKpFg<(S>N0u zatSiphH*}8+yiEr@CxMVJ!#)*lkXJYzC6Kt@WaANzpX}`Kfi`thd4gIFn?ZsvOzeG zA&+B!;uxShWknkQc=YLTWb@E)__d+oowJ_^-*1{04sDzk4xasd_+Ian@ZGgj!huy& z`1)QBZ@*d+zO%k0?3?=MVek4G;U6~72;2564-YF3kMjTfkN&^VUjF7Yp_JbDoPF;+ z%QAZJ&wRgst-O?WRJ-jvbzlGh~#UK6e;X4C zXz(-P``Y72{B{gU=$;7gXeXuUy??1ciHvjmo5?J4$m0Z>Uil+FtUn4ZHUBfwocMVn zllVoV_1XWK=znW?IOX@6lYD52=k&4ipYDymmp=$~Xa7JS>mP;s!G92r-Z4hOah30V zJRCc&Un23x;oVgq5C3$O|MbD17=LDb{&M~Mi`jQ;*>{`u>!Y6D@T#$X&OL*3xPXGa z-Q<1Yl5qRZ!Jz}W!}fpUC+O-hHw<|;Y!4c*4GvAW%)cXB9@+nmo?%;N&)fg4+x~C# zE7G}&a&ru?lgS6#A9n_aoAg@Oml|hXrMUjDzVL?8u1#DTb(05&+m2~@l?_hbM|PYw zamfK>)S)?j2c5e=9tOQG{~}NCS+Ac$KSDP>?^=8C__}lSPPC%~kN2r--RinGx{uB7 z1C1lym$>@no>P2ovt1KCRW&r!lGWWqLrr>U80tTVV+2NF42m%x6HwS+LAxjQlJ?JB z?VtJDKi*G$6Z;<%9Wx12Fby+Mf?1e@_QmFB7$cD*JJ)NIXghS@QU9O^jcNA(ZtV~4 zoR-7JKhiUf%W6BcX-8C675&cm$9d9BUMmV!CyT-YdhJ~I^qPJp`Z6?R+_She>iCT+ zzG{5MC+t(_x0W6H{>iw9c-%XDHGK`%;s39FP`6k)vRr%PRpkn*)$cXp3ght>YJb#O zN7wnAumy4b`%~y$Q`|!7gd+!9EHdOF|r=b$f9zmGUTNX+H}!3kQ0~OZTol<0#VP9K3M>;KGF2cUAH{oZtY6zf|r4)rsOL&H>S%l{&T1&&>WC0K?PSc%nWKg<7rng9P5|Nk}q|6cxo zM;B z0`1P}Nb~bo74vU;rrLJhWMO?5ZNkR0{vXX~L8~+~-Np@&ZO&^MR;-Tk4PqY-AS*6| zL-fv((#8?`G4yyZ`Sr#6Sc^j!$~^zdx!%KK??E4YlAeEuKhd#O^wdK4g*RWwvMsxP|#ah?}yQHM0@(SQNx3@Vkc z5ZgaP$>A7*F=*ed{VxyZ5ZhT@-QK@;xn~r5$i{2z|6A<;2kiel?Ej1MCbIHpTSnji zA6sWaI(?scK8z=~R|OXy!UTF5S@3_@>t~|yBuv3H^nbND%pglJ3p;-KtHc~~9{Rq< zCqTw>Y!TV7&tM6;3`^wo@?V>;NMDJ#{>>7xX{8;ioJ+WZtGJHB`q$b&+d8a0;~8{x zyN-$*^h-(tYJq*(7@3Amtl8ZgQ^V^ECon1G3xgemyl^{=(HH_DsZwsDWJ9({oS+4|Shq%i{} zsC;BD3|V!r zHCTuI&z=q&$W5rQR^OXX^FJP!|KC?moRq$@pzb~Q?>Y+Oi#(?-jx9x5JO{h52m6r! z+|!}`r2NmudNTg|j^~QP`0s37j1oEPpUyYvIx7uToH z;vw1lYirq)gSIIzF%-iw0;4bn#pwU)(_uV00hRNIg^A=OOu;lHxAH%~VxI?kVgJ4D z>|gpUH0)&m3a81qR$r|DTBh=wepvry`TWa&s?X7}8Dju)XY~u9P>%jImdpADR_P~T z`*&}){vUsS%`HBDzWiF$v8B_fM+4?bX8{&r36^06R$?{U*`Xc1o)J3Hb+CK{(!91N3lja>#zZvkZdylpKZH^UMr0hO6g^280Xri zyT+OBUq3*FxVY}aE^-gD53Es19zf=}@3KbzS3VpPK7zu!5Buf+wC}V^{=a5@27NrS zjT-0a75dj^`u?ZNU(4Cr&PgJLdVI8hw^_LTCF|ciHYaa(qU*fzC71QVlRffPg=}?vW+eMyto857oHWYcaXeYMT|1eq+A04x>9eO-e5y!)3R}6?^*wTZ*W}Mz zu8)lK2&11b&wsh&tw)LI_dD&J7V->v4tZr!!T-n?5zD1G7dg&JzTme@xPq%FFS3sL z3)&I%Dl+LhZ_;n$F5=#$Py8cQ{vQqdt%2sZhZw-1?Y?{T@na1od+U_DWWV}v1UU)? z`}vyucZ>fv9`jXw9A7jJ=+E@qeQS6q7B?RGx7GRN@a1_5bG7tJA2tAqqPUxF>$)e1UxH;=ft6T|HKcm?1XkD+m=axF?8vFDl`(EAGG+X~VyR}7K+e*fIH`c>#V~zh;_jagnb7bdC z^{&46?vd*2XVt@~eAYg*{$~R=VGEM;)&Fyi3!v8?Wg9QG&M&Ar>;Cnh|o zOdeL}G+fipbzJ(M>pt(jT=t%hdr!zuE($&9y5l{eIqkh>jN>4is=U|3-s?T@nV#A2 z-MpsX0*}`*s;4)g4ry%buvP(ExJ-Xs^yO6i)vpt=zaam4dBpWpuTuYRRR3<3zsS0^ z@^?o6em6fIIovTLFbZQ(jPaO&iD+N=q5qLDp%Y!t%I9OHKT^IQqW!@J`RIN>h5dlm z|JMIFNg7iS%ZzDc=GMo;46+2-nc5@d9CVIWRwa~eLzHo3Pq#9QoaeU%DF4x;ME-}r zPAsA?!7}u#hgOi$|G$##vwp>DGUom5*oIjD_v)7~lmTz^0r-6#Hed@Xws~GZf0QVt zm&Gu4p$aNZ!-W_kwv(^xCtYtv1cE7?0RysuxZl|Ddh2Nn8r` zXh#3Lj}lojhdlbe^(b+I?ET`e6Q{`2ID>PzfR<|miI4VwToRWRm$^5PxI*uI^*4#! z#eu|CdilS4C$Ie`kr&s!@i&RC&A&-hKN?8XyfBbRy)=-hCF^DmB+_KPa08kDtUf{K z7RKM}3%E(Yjk~yy_U*q(bRc)|H;GPk?UZ)Le>D9j(Ubm7qH*~^qUqXi63w@MlW6Jv zO`?^ap=U4tCeenKjefymO^ zs_Y{ojTGw9j4X2K|G6~CUjDNaWZ&1dEy!)!&D;5K$C5d)(jE8jk~yyhZw-1a^I$J z^eJ*EhNJ&q`SHn7s8ptnA*;^YZ;c#}Bz*!o5w-Gr>W=r;B;Q}ucFA~;=>4gDE&aZ~ zDhmC7S7g5xyxyf|)U3ONlkP=eksIrd+zdHT&Fb-sN_#1-ao|4h4^9_Q9) z*r)U8o$B}85Oe(Ki_oK9&7(V^|3Lhc`Gxz{Q|jP4^=0~`dKV4%?0fH=WmtigSdBGU zhYe`o$o}5yKgiDawJ*-Ax725OvPZa4-Pb=Ci-W!SU?6{x!hsD20>|K5%(bP0H zG**obEjvet=2y(ENqjoA-WwhE_^*99fU+8SjBhKBH9N-dP&s%|h# z`V)WqVt?X@G>)O7_g>=7XYVJ(3U22GsQyg*1B2`73{# z|Nk?-S^XDd9M{jDDUARB?66QQ+^798p6va#F}?EYgm}z9^9_-cFa>e%!D(cMoI#c# zn=#IkoP$pO&)i&PBiXf7*+}MvD`#7)(>;uG%sebWa;r5w#V?}Q?)2WbKOL6Pm!Uzp zUN}wG?bY8h@|m!G>@%Tk9A7%ybcJIckFQ@zUyU`$9cLe-3*E@02i0l*f3|RnEnLes zt|Qaz_?00TkLFU)Oe~*CUNO9B@pF>pw&u zK^*^oj7%cat6w1FzPqCjUmb(IxTdcT7`pF?4N$Bg=aqr4UWhishc{`D0!4{`r!RrkleA+l&Yul}zY z7S2fL94_Dz3giE)hJ`Ei+SoT>e85%ubu^4qZ>YPTjQ<~|KGfG6>%*IVzm23xc|1p_DLi&!ntnm)pKsI@}BAWm%P`J z-m`by{lYV0lr+Ym7~?Sk6EO)>kgD=ry@TqLp7UkT^|0sa`PPY#V<-y##*ykq)XrBo zE_H9Hr#H;z|FACcH0jJh3Ci1x!YpzQ=Ar$O{*PBYe{`Y?zdQa~{|B3)`F;H#C-r|^ z);1us!fj;lPuTyTH1^*be!Wjy=U4dJp?`oGlY zh5oO#?D@mSYKY&2%zkZ;o%&@^QLP>JGw)+L+r@pwz1!j#%1Y(seeY(qWIY*#iPR7USrR+D?gMIXXG@&K~*L*x-eA3{!@9@phPCY(fGxJS4v zqYXe-Ut@pY(!W5~K2raaPx=>@i+4^vnvq2gd7Qv0JX!ze5j*yt`u~>tU;Uk*ul^?+ zUr_(Qr2Z#cW~%?mjBu7LtpBt2ch~X$mpBN&G%irVEmo%b>!(iFZoU;`%ci9)<1kw|L}2r0cfRX z-j~iD`4jC4`5QTO;`jG&yD81vxQqL!@Gc*c0~qwC`gOkNtN$eB-fO3NzVd(i8vhm= zV*WorI6URJp%{)4NDgK{$`7OHwXez>zTYwQVl=!aZ-{%m{?B-E6EG2zFa^EVtC&U> z#yj!_75qVD|F513GsKr*7V_qBmH&thO>gO?|4{pfJ-=^pMCx+zQLca{#-4*2J5f^o3I6?C_@GMzs3J257fvDDfblbweGBY zLp|d7=UtB5gMB!FLpXwCNTU5U?+3Xd-VdJ4|DzYy|H*tf{*R1n1-5SXevzfOy{iBJ zM{NIhwf)=m1HAs!2mM)7KWKlaq+gF_T{jotj;{-b#}UpLu$m~WfR3b&Chql?3R={&>$2K}A) zzSuD{i^Eg&&R6)qHy4MY^x^0c&M)QvUdaDVju2P5yf};^#~}GEe=0d1aSTIBI<j(SobAdN2W*ef4k=z_nboQd*UyOM?Jmac=7M7|1&|F71p+R^PARIp-;kp zy8h4X|AqhOfjsZ}rbuTRW}pOF_my#fv*?{WeOue@13;gL9^t%joa5KESHG^fMX20p z9|y8(ba7Zlu0VVib3D_P^xETm-0yksWZYvQ&Ml5}j92?@4c1`;Hlh5Ft?f^iq6`(- zg+17ZLpXx`H$JdQ3iDq!o9hyajK5RQeA629e(!(!`H&=g>pWkw@0Xq{*^Df5$m0Y~ z;WYZcY`!3Q4j0h(E<2ix>;GIKui`px;x^)*LU+mgc!&XH#bp*-7jUQNH|~RWZ~h0` z<-$EL`VPc(Pt^{d&i^-E{zq!J{J%r~SAV9-dPEz!u>R9MV-PXi`4z@=zgaXmjG&J~ zyZ76njL(ss%e|AiAJ+f!cn=Em|EtvhY5D)C`k!8ypX-0xwi*{Ijbe<)1Wd#vOhNzO zng6Z*)cfV)Fip7c%l48WOE3#_P&r%qNiM)5EJ0FSm2zzvy;dGi&9$!@eI*)RbZ_F) zQ{AikF6^(XZb-3(YuBnD^fedu*QGbCGH=Z}tFZ>_@KOKI2I2N=?C%@u`3LHGbkXD3 z_&7$OhrQi6R{iUk=3DAtvh|Vrmk%L}Hnd!H?Ela|B&{tdMb>d~|GP4J=QwFQrZE2R zTgv?x96#NEO!i-7g<~rH*Di7o;@E+GzPIYz;7wDG|*YwY$+c8})@RPj4Pays(l5?Lb zw8gi{iQ0c_dKje`RC0=lSksaP9gte>)$&!t|M`q%v?18mAsFv`mbV} z@$`SDZ|$VEBp$D;)iY8)!b8Ump#Lk<{-Qj$OB)|UF&rZ>3S&@=*xx;#?EAF;{stS; zx0y!0I=I0%TRm2O9PFD#t+?{)r^5utPsAkTyM~1+j%yuW8c?yL|^$l;RRTPWmtiJ&uk^R z8kLtn7S@mzKacYc>_1LV9yI=MuW^g?TK0Gfo9J5*eX8+ZV?cf^#uL#{J#_Q(|EFe=wW3Qst2=+ z*oCRF?4*(GA~t6l^=#w@EOAcR`)qu+@-{Z}Dd97yFt_o|#`wJXzma>y&b`OJXA5^P zW~VJ=r_E=7vuT^~#Qq%3u0wVhJMSX@HyiAn^e*5Mt{{1q|J!d@>9J2Bh3oX2Xt?a! zkVagq@3y!W*L9b?kL-5i`N;uf4(b3?+xx}>+{+<87<9GKz zm?+KUN%!yGCeds43&i#Rr_l54izoj7c&*d`?X>0x{qg#Di|9R{nI4vqsU7m;E#K?|-!2}n zf7kaFb4`B8c2!p`bKDB7#A>X;I&8ouY(f8L;+%Km8R)%#;n`>8`Pa-5Nyzgn$&fT%9oZG&2rS7@EVbL$DjWk%ELY)71#D5$^3iW747CGcmSpR3HXL`%Cz2+G|@~p|ca1Yt| zlJ?(J_MKx|$X4NxuK%-5d>r!^_do5o*FZ(1`3wJ|-&5NC`UVR7|LGezB^>8NoF=2a zb0&uM7sUOU&e1R660RV7(tliLbJ9DTTw9IzO}~l4`aidDdD8WL%prsm>hqmOO=z&JP|cJRBns{qLj5F{t=U>j*11i|OMr z0TWSOqy9yTFTWOXPJbHp34UaK@RJ-f1=BDCC76Xd`2Q%o|6rT5Gx7K9Xe505NHa~E z^wFV0g>RUq!i}yzr(4%h(G^`oMOUck3Kc4R!wnT`sL+_>2on;LBO%R%lXK?$+WY*n z_xZW^4izf8LWLS?sG)M4p+bc-+;GDUb6;y~symsv^Syt(>v=wFe?Dt}ey#PaXRT+g z&zoqQ5XY^>IEZKHM1KCC_=+rY2gG0Mzi(P-9NT^^V~krD8@DFYXdNoewY(*txp*53 z5c@FV_y69ZckW|<%dJ01Uy2;Nn!V5d-e!Nv<WR!|eMn@jwD=9mBv~r%rRuhFMo*(s zIjhyrRd-GvwzEjuZZK9=PbQB0ZKv=4p$2g+(uRx}WhA!x4q*IEe^(ujIraqJ$7!6! zd30`Y?RfGXz}zMG2yqTx7h2l&pEiZibjZC(xaNhf9Vz$c2rkOwGOpknHcw&q3-x8v zx6rfo#o-qH4z_-yI6NR9q3jEzd#yR#U2O=#kg+cVramwEq<8<_4 z$Q3z1WJWnZzx8~6s(wGdf1`TwRo{8G&q$2MSd@OwT>77xr%s=U-tYV$VKO;2>M!sI z$P$!&qeS_7ExbXWi8s;vBeC;jtS{!0F*fjRa?ADCthw}B*mV1~P+_k4_X|D~?zyG~ zetQS`@6StLOs_M4ymm=RSV~`xhJ_{CFC`&iE?2$Z4lepa_{TM02=5*Jd^nJr7WP+8 z2|rz9?$^4P!@gCM!roC6!jHF33O`!$xlq33bLPu_HvF)1LioX)&xGCdro!h#W54G^ zi+M@S^Pbl?@#-4AaQMut;-t~EAy2_ zeXS4KH~iE6kB2H_VKoKMhUyi@zfKv)K;q1EkH^2x4Gzf*&sn!&O!)f=W5N!3>_XYm z&xLJ+Cxp_XabfGwabd@}3E>~&dHZc|+^6W}@O{_%y$dggokf$vo<&o_4^K@A`~7w> z?z3oG`0>6kgdZLHLMY$hek>{uKka9o2ftOJ7D>FPu4*RJ=s*_7Q06(DAm7JnRPQd) zKjRr#L!hqPJxAiYXKd`jzC>;vA&$*keE;$>d`RNS{X<$Gv4j@(MZw9k}b2W3rJ2x{FYD- zT}QPI&~aZ|0h9eU71a~0KP28Dzy44{y_6DvkkL;>HeArZp)PnbzVV9vOYjD^Su3XO zy}qGM-PeweDf-#fX&;GyJgM#BoSM))U0s1A)6}g<57q|3oAT=Ud`WnV?5-~fbIIQC z==UTS;2p&8k1r;>zoQ(Iy}v67%gL2kjkT!Y>#Qd$*Z3X^xdm19GI9s%uBwZXxTJlB zC*%Ls{LC7*F2>l`6|+lY*rkQ+|2%d|J7A_`cVQ3qVm}VzFshLE|2MOd$os$B*+yg! z@qf`M&!$5DZ$G~7P5$p)wzc~M|NoM-8h+}8$M<+sk@{YqY3VUWK1uI9DQ|YRnO-Ws zAn*UP*_mnVFWVZw9bO^d^YOj2Pt`vAlUY<=cU>9RwNpRA7qy4K$+oc7dH=uMb#7pP zop+7jYaN%5|EEX0oS%P};G4_m1m4GKoW*%u#AURxuQ)SZtqsuHgo5Aw9)D?$;f9XOZi{J^BOWmbm71u788;$0KPK^R&gj zuPuhScJe@S5MtjzjGe5*5Zl9$EOz~)Jm+by6V+^UjrXB;ob|U*Kh67(B)tJ695WK5 zF&+~!8QC`D&1Bn6{`OYoYO`{M_)WYFe>dCYS)p;hHaME?*P=as~(aRt|q*5-@tj~n#PIo?hBE&3fi^#`YWSEWDoe~Ulx ze>d}gck+L?@_$o7e-{6LI{*I^ z|MwjKmp_~T{+&Ei8U8=n>bMFrmf`;L8i+xtD&+qT7!-!k>jv|G>1A>Lix{kibh>SqRZI+)AMtV)yw#xsFi0O;+%p6 zl4!t0$45zg!(#eUH1u-=>rN@rRH5`?JrPU+(wpXZwab|jVjH!49C3pif@g`#b-`2iE#OQfPY{xYM^6SU2 zld*o8E3IXcetxp_*XHz*i;*7Z{_(w*(&L&WaSe#&^p(i3|K%DpN0ooF`nvcl)ZQ0= zg?f4dNw%Q@*Ieh@jxz=n;`h^5)7N4>HlmFUZoj}rb+iBI+RZi{V58XA_^qGDS?nL0 z7qWHi)REb2+ims_t*BTf&Okn8*nwTBS|1%^V-CV0u>oTDy7VvP$SkT0*#9y9V-!1ydL$;W|I*?* z$7#oYxPPO=b{5B?e3}349yUy*pu>kscG+c6SNFdI|xG1eRK z|JiXty9Bpv-@()PqYd7HHJ<-*&;O+7f5G$Dzf5J zMxtt+=kL83O|Ltk3}Y;PJf6lMdGF)+)t15HRHbiP!tM^R<~Ke26ESdP2}(aB29DhJ z4Q=7?y%=WF-^5#(i?^`=acnLhe{jZFneD|$JHMsma&#Vbjdxw!VCCk5Ya_G63&Kjj zt;Slc$3|>H8G4QB?;yMR3A@N1>-g;<_hLT|BHzEGk0+Mr!?v?uc`=lJ#oGS*ZhOT9 zckAQn{*(BCSFPvc_*(lUQT7w#`G0sZR4j9^jk7jOOQQpE?qsa{vh=zE?q9L`lYRmX z-i7$>!1#^%xCUDF?xIk$&l>-Sib5Ug&lGvLibC>SQD`_>6sE@f@IFrCEY9O1E~9Oo z`S0jBW&ZndWAc}b&68Q=?v4zN?M0z!XOS2-F|MH8zDGVl%hi{{BeKsAwVyE%>6`uogXo=8T-zvd2J~Ubt#WO~ z)ieIH%YSTHW~}YdOJUP|@82Br?`vKPBkVU4mGs&r&dV|Rd0fAD?_GCX*Pwop^K>7o zCpmxAPPZTO^YY!7q+=Q|T0Ubj9uqMcQ}GH)&^FHaGdiXz=Tp3!6O`*S+R}(~y`HZB z?|HZ2h&n%o^llhht;;%B1vz+lX1Vie?8dykM(}<{k1VMatlg-pik^`{_nH;B>qLer?j%? z*xP?L=Y_rpEsM5Wg=tf}Ya9m>yaXM$$)PES|=X-F58`-2*arQ=cnaQCt@aF=hvpUf+L!2m-=P1UOe21O zx6HLxOfy$NUiau#3-!m-=OTUssCJg~!rQhNAb$I&q0%@3nMm=sRy?lv;~Jdr_|;S2@1f)9)i~Xa8|zj`D{zS`o(|s^n9PB%1ML{2@*6+`xvf0X>Uc zIlF_5`_zSF(&G5T3G#i!@rTpoS>(qb^|-Pa`s;vze!uaIkxqc z!y{zMc8t+!wcqA>`T%D6Z3$aWkK+#a4k7~Zly7jI($ zI<&ny(N@_nw43{o)%MDvi=NRoYnI-UFppuda)dO!^?-Q{Df1ZU@!L>wjfm<4%5k~( zmY(n5TIKoT9sjWyOR*d)u^QRNXTw^udKCW`wWuq8>{BKt@G<$8ajyML`>e-CY(W`z zU>EkF%{YHMU%P|P-MNB)y@r2nY&}cn_}Y#9+onVOU$o!|QZ@YF-TYs)(D%ycyl1$d zJcx9{J~&M8T*9|@URCs3RNdtpCWa#xbYN zar+}*{&%|ma@T*rwYuKF`2Mc{I4_@zxQr`EuW|iX#XQhE=eYh^gTf8^E#&sed$<3X z;rgZDkyg=U9R~6NswNEzkH|iIUB7!>Tj;um`aYuVfk>V+uIL==rN=&$YGuD>f-*8j z|DW<+PbO?9$p+bxvSXC|(S^)7^~+#&04i>p z`~Qo+Ve66qGrVHI5>!d6#2fUw6|N0s>}$UNNF9_!Vu@>&KGQxu!F>9!eA8Msp9pW- zehYK)HV)N16Mpv4JdvYf8Vj{U^g;LX?R(To3;g~L7NfLp-;l*pdi>_sa&jeBqepvX zE!q8LZ5(nVwxA3j9$(mDdl&ZL`@26CdfzkuT5FTF<`^3J1xyCLfG2xg-~|&g|O}R z^P$Rbwb(N0#qjq-UkvZn^bJYd%}ApIS?oX4H~iE*svlY3qx|Z}!(RM2_3?1bZzr&4 z{*g6{Z>dG_*Q7@{$^+__@?p0Z-nNeZ}JVkVT|D$;j}!@;yfJqViuAkN=fnR01KW?#=f|$oczhch+K!HK`akp;bdBQw zBa0jw)xk|@Mho)&A5+xh#r%JL11&`#3j^hqU;o!!pF#Ba?ZlL}(GYrPO5PYoAA#I$ zd22`H*I%maA4W>6nE$abnjDKNZLaa;L{tuTUG}fTWZP4bJjFi9FTajMH@i{8|7S~T zGyH#kV*MEXe{=N7p#iTrrUY+bCQ4tmhBNsV=Ax~V|DWI^kewMm0hzI#CG-COB>q2| z`J*j3!iRqHeVWz-{0ez2z&mJ})-Nn3Q`+Qd?UJSRr_U*Wf^ z&FuX{G0XIdC2Z(dz7xv+;?Yjm4-TJ?qx9Sr_ zS=|37AB5b6J=lwn)c^X~{@e9`wfR2W{@X8)gE)*Tbib@U@^y0pKC3kPU1PxShTN%l!#(>vKz}?!pP$e%5Q9*?Vq>UT zxG~hu+ZgJ$z8mT{zZ()rq5+N5-wjP@MhlLtc~=|q@&C7yZ42KG?GxS&9b^1|vHz!M ziv0gj|9@B8;o-Ysi2oUe5g3W8Q5&t_zcGxa*H!v&jHQo9!)DjN+x4e58VlMOCQ6%( zsdxn?cmp%>Ci3eo>6eP_l(%fpMX9l(Y@@lH^y-B8AJ3x}Pvg&qYo`xjr;+bB=qt9) zzVBc$mSQI5$S#2A7y0=J(lSXl+kw}$NpxgvB8trVlw*X6~o04`E3uXF7*j}$^EFjpsd|j z=5WyVVI((bS47)OCe}FqSA9d3^jc(pso#-oMj9RX@cql$K88~9$nTS<5$EunCCk33 zU*OBu^`T$HWn95E+`ui|L67+b_sH(ot(!?cifNyDH}u&jO?}ay9Ed>}g3@pH3&Y3} z7>Uv7ec|0OmK=|k+kL`Baxz{)3DVcyziw-X&^uT04VP&{`i@}PulqiZ$DFVHsR!a% zarf)*hMD&3`NBs1kA1_I3(C1~0&J>OuIO*Z-+tlUA-n%s91r<6dP_Hk1!S!{r_Q_c z4t+85>$@D#zSI7U>$@zKRx!_-&g4o|&Gx*>wWyq7?vU%BN?(tS*n%?bz%Eqp<5!^e zCOh(w-MGrHpy${Bxx~J)K}`wvjf}DEN9fUZ<-hMU+qyDq#OH0$_fB@s(m%jfWss%E z`FSlf^ojbPz1WX~NY9oB4%74N-%RI+$MHX%&$*^}Ec@s8xc<$Sh01UNKWv8kMXws* zTpgE0Wz_pQ??UGn^Ns2BhP&+iP}kx2{QS6i_Cwuv*ViOHWdg*3f&t9$16^YXcf%eaErHb_m=k5BJh z#{Lht&JX<-atqmiWahj6Rbm9BRmkri`2bbx!~~Fi_PhQKuFJl46U^hVJrGHGHK2Zq z>nw8p=RO$!_rBJleu{Egtc^@IOwc|*#Yg|3{u<=?A&B#XvfmaPKz|xA#9X)T*EM~8L)-E z?Yq`juui}Z`Y!bT`eUK@m)6f3_e|JhdoT9mAgUfdZVw%%$2ARW&xmKB*P>yA_7D=N zKdS$qZ(YMjt=-B`R$aV6c<2rxUO|Boc zr|frJx^qpCNksoYKmY5t@nAe~e2lr>cEoy{^gcgz{p#8L`j2F%GT)^>&OBt__bLD7 z%KuK+Kb`$W3yvU#I8V40EzYCAd`exLG>B)vfyE^*8o zn29&>7Utq@EI?a=|Gz~0V}B%9i`*uQTrSHEP` z+aJ^~@5pa4vj4(=CRd`gp1mTs{Z3u+Y~Qe!z8)K~1##W3GI9rYVGrWG;W&?YFTHcN z`T_gt2ay}2ewg+_`^9x1mR5yYB+>hoPlRSNjSlqu!23paTX*vqc>?d_G|nP^i(|8E zj^E-aeW9OkZ0IwUo_*fB_GHg*o(WgTZvD{L$gT%M>S%H-#$zI?<}3em)SdLWra>*H(qBQt80CMO z^3NxzFIN8dePG`c>_kTW$BrlHNiz0-pLb4iY$o=tZ94j~P$G{v5a)i*B-_~WcI4OD zW%s+-{LCsgn!V5I^KVq2HlbPF+QPOzwSVjM|09l3zbT)$5aZs;US{XN>Rl=BALdHS z$GNTHOV42Uu+;gyExl7(>=&I5Lt!(lEwwcT#m(o7E?)uSe zpB5aUr!KMoC)t0*IgxwivmXa>80okV-@J<6>D)U|OHU#lX2#+ zD>E_AuRRlvktgszPNQv&^1neDBRiGzt_#W*vh>_h_CLe_mw)qC{=af~WS+9QQ28fY zZO8bZv+^nZxVAfa5l`ZOF4Lc^pLK?oA&H zlgXa1x$oMX@xE_C%LetV^wbLV`x3GF^s-MEDeva)-)A4#ko@@21$B56`+(k`g-~L@ zH!u@#;w{WY{N73FH(w0hzc&75JI?>>`J!)m(z{>Q<|ixUxtQE4E^8^d997pn`);ub z^g8EMyQUzlrmsaqG2ar2d29fxZyNtMUS2!R_%rIq8J9-#fbr+u*8O+PdThiNltp>p zH5)&T-zsgZG5(K^S;p_tHQ)IC4CD8+jo*`vx6S{*Z~p&LbNz9o-S|J!3FH5j;sXX7 z-$yfA^idu`N}pwVq4E7WFNL;M;se%+511~0bd|F&2iO-f$H(2_eA1rlE^-g{Vn53M zsNdl?#y9nO)`%;pJ>&cI8Q-T*_;!IjlW0JTYd&cI!>B?n(ylu-Bi;k=U&^(iIi@4G z)A;{pe$KBVH?sK>MY~O0npk}kU zXVjq{iS6Q)n|$j8=N)qqmvIHxa09n+2W@O@J6qd9#ap~+v=qkEJIART*xZTq$;b^?ekUlu=+cKURa(VP@j7G);>|^yEYIP;|J_Sruk@bZycG762hlC= z@^DQ1h8P607D@DeS38JIN4?%VO=fWnC-6Q_<1Egj_isjqi)8n|jtrN{o)<@kE94E_ z!X0d0=DB&+6<1xy2L0#cJ?W*-_YGC_7`ynucHL(6fchq}O1a;s4Y)=LQVZDs%D`H}d4 zHZX-Ljb+c|QO za~KlO$9WASM_?pIV=StNx;{3uR^D~;ttS)mO#07;cK_K=jD!6qVlt*8?|(~wg&x1D z)4^tz(BD9AKKnc81OI!8`4Q48rfb8HZz1~MbIDRZLO%X}g8MqfF~$)WNKZPS*apeR z8In)eU$9S%)30CS{#SZ73D1b!bY0oRmWRsoXY>cY{0ZN;66+wZ#b`UO{GU?($<9m4 zKRsjLe0;|N?e9za|Hkk?yw5G8_#a3ijaKC2+O!9k%4<1t$NAXEu>Ud6t>r+!uu}SJ ztVN9fOZU^ib*W!iZ#%zkFWbJ6z6Ht6+6~CBqp@3=l2*Yd-a+m{)pRk%qwNKbkWv{Lr}Bo=Kk!yPG~4 zelYHTh40V#U!h0*Kxtob`Y(z{DfGOicwWVxiT}#tSd^dkT|DwMdcP#@XUYQx zVi1O47)GE%`{ytYse6B>483jt6By~Y(HIMTfMN6QU~U-uY5cC{fHmOC&CwpiKQ87U z^DC2RK;s7fEnmI4A76fg@$G4RSLD}mv;Wie0gj4~M^`)F{Q#e;Tzov=xTVBm@ZRV6=UHSf$m_Q60@$Zwo$nBqYAcy1QN?F}L)uJK zp5g!6w=Q8Ff7@>%DX#|9r(B2gXzn+~xYXxEO zVmVe~HP+(rjJSrD??liK%{HG}zt2Avd?EbYe8kEP#?=a54Rte$L-nj_q2}r|b?&6l zuwhcDUuZ5{!OPlT`X8Kgoa0c3`r*zC$wKEj!902SZb2D#U>EjaFZQFYNdJrb++p8N zva8s=QRd?qPwtfdhk2gQLf1*=pU)Kc)_zZ(Ps`@Qa8N#nQHvzf`+Wbww;8iv?i)Jk z9rUvA@>{#K>AR&{4={7z_a7a*Wft3WywJL_`U&Y()*fw+d7$!~_Xc&+I&5c=Ockov z3PZhP5*gor)c&uT!T(sOojy;!u}-@j`TmFP{0+yP!239j`0bjrx6PnS2BS;~QR$P|H6#`*K{5NFgM z80+})n25=kidRsA-tUT8_?-D8L$#M|Z~f)}3^U0$Q8i8cgYx|rz3#Ysi@Ef-(Lhfk zp?;6PUUeb+$R_3ciRZCD>{kNGc6JKG9J2uLU@?|rIaXpd+GexQ^VsJ(?DI1AnNOct z!alEHpUK9P>_3_>@paGfe^0UhYuNV<>^m9L*2-r+Hew6X_wAz|FQa!lzYgU4|39bv z$Nv8T>~8`8myF-stk}&Do3746RYuIU<8~px{?9CS6?<&&MRK<5x~m;PCT3`VG&w&s zpK^XjogY#swNLzioS)FvPyb$_HXzwqq|HZWY-h=wHsF4F9>if(p%%%gdrz9l>Iv@e z6!#Z#9KQaH|3wlFi2eU*`*k3TV@OYO4%5W+(L0B_e!NdVjob=t3GI%|8rQ$hyZ~vj z|NlIB5mnMJlUI=M|DWT3an1G(BuDw*an7UIdHA2d=>MOiU5z9faLX}wko~>2e#l4Y zbJ+Wjf%tFx|0gMbXr7|{X|Mlz|NkcCb~E23u743@00+r4bx!%@y9}Yn_@80qlX<&c ze1SOjm_-iToLAY`Uk%OLB_r%N5(bv13l)eVoRJ=l`9x-DaP5bj*5<|Nk2Q|Fw`ocJ^x_H~qEHm@q#*Wqx{-`RO~& zPv2*LI?~(CPe+B{;{2EM^16u2sOndu@2VtRq1UZ)Zp%s@mw*1$)Aa}DJJ%&8;hKF~ zcE1*GkhhTb-*?D+NVUJFU&?uHcr84z{Rmm-6wfPvp7Wevx$`^V{A!$^|2%TY`5}!~ zv|V+5H(eju`Oy4%dgf9I`)dBYd)TK!S-?OH!Vuh3Muw3iP`%Lo9rdC6JJkJ^Px7w& zE3XRqjI`fqjKz3V6}f)5EXPX3_Wx>fE!Ly0i2Y?p z|I7J*+x7pUd7Sc(BWzO&X?m-Ef!P1IQ9fHxh8;-H=U?dC+ePo3&3%_Ef(XlH_LX@J*AnrT_M|d^v8Gmi=SjP^Pbb+n204`Yb=-*V^jw zSo8m~x{e9gvLm z0Al}ppPzewFc5<<1j8@_y`S}NklnAAgwbTr7Yf3raq6O>>PydkytFv~FUC;B*nx?* zC*#Tdzp3;m^Z#CfIOkp0-<9Cl$(JeP0r9m@Hr^nCo^7p*r%uSyxe*ed>kUgunD?Ndds zMZ-gRBZ2zObZK$^Uo)AG6uTBOix1ELJ7)VtOy8j0vP9ia=I8%q{=et{z3-UQIE(X$ z<^Lji8CURh{vV(JBlG{7#OL9EXa3(c`P{%Q+(Fg4$MgU0(Vxt}c|d=JKcD}%d4=+g zK9%lsKm8tGeKriF4?=HTTVMO2`$cmMZTI}?V__IM0%iZg|F*W?e;5BZ(!QfH7UMAy zlQ9*q;3M&WC*9Wz?klqN+)@5lO8u8n|G7Uc`1AQ6CGv=U2XBxwk$v&m@Fv-7eZAhF zec=E9jrf3X=>zzQxiNm5i?^`=F@EbEaxs=-IiAG-t)xGR|65I8kH3unD-8Xmuf;}e zL4N&rep(s*$@=g4{y*bC4MX{+Nbu|GvB?_yo2|3A<&gFND+h%gj!7NjUz%&bi@pbY z(J@1OfOzCCGGm>9ESVEmUCl48;h)wn;{R^o|1Obt>JuTE`9x?S_dE6=4x?ud@C>Dy}Y2~_cs zD{oqBjs9f)fz$M}XwWy2o}$xP9WdcI%t<&Lq4$$@}{MQ2MhF?%DSN9-&Vadx3!%g#WhxZ@T_J{Fm_u zgZqRb@)?E^7>V>O_7|h+om<&ojHQo9j-ExPT>l`pzB(er__B$9n~W-BZBxltP)V;n z%CAR>?KhC5H{i)S+p*0uQ~Hy3<(u@kFc%9D*Z+HmT#Thyj+I!Awdnnw?@rS?F zr%&G!(>G~blii-d4zlMf`tYL^>%S+af1@z$C97BIe?TqjYMfU)+l(Z=0j2f*!hZW5 z#9>sS7D+UtZN-QD!{N&R0Oh|}`JbZvUr_$FrJB%ezZUiG)9=60Tjv=+_?o!{)@O)w z`(yhfEzcPDvss%Zj$L%vj(zx9@>tY;D=fA@dVeEs#&&-F0rvkiy+vJfmP~CBKfOkq zntlHQYeOJnJElmHLFM z%KL>o^p@*l&&UUOgg*b|{pn^SrT3?IHnB&CtVu&3g#7n|ceAmTY;6tu>Az}5v2l1h z|DK-rQ^_HY8HN!UiP0E~@tBCVtNI`G>35Kw`uDrYjO{F$(>IY{|9=Ym$S%gYc1K)S ze*V2{Yb}WL@3jrIr#7dwtuR@hRfnwqFMTS#PJ1G*|6lquvGBGV;@E*UMMD1d#wSXo z#qSE`*Z+UP^Go?2lC+sfWnB9#{RZ>2ziq#T>@@csT{Aofvf6poXfx#3>6yc(uVUMg zY*+pf>yNpPdm9Vz4i;l6mZPo4`-d1~+KH}p@8*#f zKhEgim(Oaf#d=iTW`FPZ3mfTi&6`@~cniG@ABi6;us*)DUD$)Y$iKJzk^S`e9v>tR zBl{U+d1MmJDEk+21=`J{n^&9@#&>7$%av|K30J{?_|jF3!jHRJ?)`Jej}z2E9(1 zk8`kR(%(eGUADz`{PuaAgI&GC{PQ&lk-E_+MIoWc__&=GlohDoD*K&7cST3KH zSdFzv7ug5v>74`QZ5(|geG787v$iv2m+_zY{ojhFkzvz3&t-}Jzxg9Wsr4UnzcKzs zubtt#W`8jM{wJ>8c7yHud9IV(Abq`t$m?WsK|jxG^aIbvXVn(xc=t#edQ} zSGoRW{*T^_7*qK4`v;5MM`;yg2bo3Hl9AyUc>l-+?$UFG{r19Ya+55}?Cp;p5xM*V7Q}owi_l5D{ z2eZb9A73zDwq{5uFEn0ubYl2vk@2+Y!@|BwU>j$B!;E!TQ z?uCZ#dm&lyhmaU^C)D4(6Y2{7J=AK`)tnRCd1;Wb7~i)#IZ%6UpfQ<`hl*}~)otHS zyDIjl#yeH;Kf~XnKYbv|CjE8Tw*9X|>GuB_w$7LscDR;*==WFppW^&M-@6(7$?$!A zugUo1!eL=gvGK?0p9-bF>K97G$HLw@#tiYJIiCvU(?*2-8%CN3r(aXvL$Fz0v1wjG zcyGRbMejrQS9}ubBQO%9F&5*|`_Ci8M6&xcz7aw8d_#N@`3g$#20E8%pUqdk>2D%e zq|QQSI9tC;opaNg&eG@NZ7e|A`J|-3L$4_G{U6V9F?~6fqKbX5l)jQ)x7`?4ySRur z{{J2A|A*#iAg&Qpzq?=9)Z}}C{OB$E-eY{))Sb?H&4hY(xhe+KDb?kVTFy zt!6K4+S$J~?Eh9a^#I$-jwkieHw>4rW6Q7uyRZj)u^$KVk@*Gv5=;`Ud{@P;CZ!UBbB`P47VbCP315jFGHIR@yO~!25V= zAK1Rrw$I`s^4}jeu5_7x1=nx`)jP%i?9=|gp#2}qequ-_SXr zZ)o#8+T(Mf=g@@=ns>Ty+B%IX_tEo-Ybv#BUmdyaKB86|t8Sj>ILkarBrkbx$K&{l z=XlQCM&~#PLof^@FcPCN7A*ya|f8;;?{&17_KN=9%-+N15bMZD7;2kW+ zQnXEC|IvZi*Zvpt|ITRtqZ#p9j|_JIkwz=JjqNX&PfzTtc>NP$B|VNmu8wKHFg8Z6 z$42xT2l&1Imn~8Mx)=g-2X3{$m;q0UwsF)s6#yxNTLD9XbzcMDJ$`!ys}9hM|X_HiGPa!MEMXO+&q>eBRhs zHQILd8Rh?i@;^@bM_f}Tfh4^FS#|z+`;}=AZ>zV~6@4^!R zQdHhm{x0yVvE24b{Q3O9w(IIIbSzVUp=+J`OFJ&BU6&&po5U}+i(hmtEjV&t{36n> zr`7dT8}qMO!?)NVPLPaq`V(iwPoe><{pVV&$N3uNh}?oQ?7*gn*83aac?=I>m+d{s z{*AIj9z>jfaG2coM|Ft4fGT<|l4wSYYfO_J$l@5{+AeX8vlH}A{j42ptm{TUja=*- zKt>y&3*F!PL^vzGr_CIdUzk7eD>0w8FXD1cf7Sa*c5BaGBTK*iv2cUz^ z`8}t^{n0m#@_u29dN{5PR)6w?{v^I$10LAt5o)h{-@1JZhTf;%xxa1=L$d5;b{+%i zgHSz9y^h*K^*q~J-%q_xPm&GnY2yHP0nNkNiNWlGeN#yL|JIZG*YC3{=s3mxAN`=; zU3$JBzR59_=DGWyVHkmtNGIfh(e%!nj>TB|c*HhC7MZK^l0H#d1-mtwoQkUJ9}BOL zC8(s=);ylq|Ay_ENbYmq$bV$(>G#EOv4WJHuep>$T)txhwS-k->{e5kApaj>U|&B|3c5BpYmVid6D_= zPfTI|zbmH5ezi!V8EJGNi?*Z64?56^E@Y5J4*B)}$0+}!K8XLL$Jo{<_UD*9PM~Ec zKbws6eovEUk(PFzyok>0>VeH{hw|NpTsIpc?XutEccv?k+tx;`py$_fcCX^vI1TEO zXiF27-bK{WW1VotaW~NWdEXr(x4mE-|8su|cj))<0FTfop>D-M3_?#Z2ZrqaZ9y1D zcBI7pWXwgRk3?MCH;1lDK0Z0t``u!n(S93?^lov<MlK10#55sbL#lAja zC0V&$`~kTZRT*{GFa9KM?oaxbU88*x^IW5S;)NFp_w@0B4_57>^lI@Q? zyGP+|`K-rAY(W`zU>EkFE%hk0XCCSQ_z(Rb{}H;7K^8eQo_rLV(0u8U_75LUo++ev zKMJjA8T=^hmCt@0#9^cd*r(u;xh&qhLV2T>oGo zP&vbKvs@QSKl@BLAuT!0b)nvViRq8bZ&m-Z!CTPcyxzA@%6X>GJqoAkWnW`=(V={H zDwkc#XNJt$j^B4YYoGJDh|9Qw?q56%*T|ml{VCiaZ{ZGlkNhdzBOl-q`XrVAaqd5A zi=F2b=jB=w^WA^v*Wmp7+h-sKVF-p{1V&;sJ~IE9-Fe#oH{AIR_5K}a|6OA<*>Xwy zpUi(>lx$sB5XQ=5JSL)|{b86)R-Snns?I$OrN4P5Ot+mCgA@Az>-u?jyvqqPX}f{k zJWDyBt`BAae|Dkx^Wo2g8IGBSs=>3za7`>nt#tihA`gLU*L@p~KSn~|HPPH;?YLw6}ZTcvHsPV7eS z_nrym z?EdPbaFmSm|BjO;GvJPz_^;Og ztGLiV6v}5XhGID48i183ea}0`7q3O^KQ8iH1Dl;}*9Oscs3$jh|KdBiWxn`sGENr#hwZl8uJS)`dd6huL*<{$*v^u<`O3fh z*kl}~87(-1_+94sjlot_pJD&lqgXcUl&M%Y6J!z%Q_X)fGPdWDC01lxNHE0{7 zd`@tG$xh|9i_EXjrOf5fI9vH1rF=_o@%s@nW&bqU>c3jHioyAnIz?V_4qy|RmX<09 zJ^x*=?dp+_8?S#)p4Xj|x*|h%Dcf6rKQK)7PF9?99`>)iO@u4m-=WGX$QcF#t+&C&9wo0E7ui-S+-{*?^m)5_1cz+ zamJs-g4c*0uOC^5=S%0u_viD!XYs$0|IQwt zyYaU8^ZVk@`N}Qr<_FX~j_pq$F#mr8|6d$4=}8nyxOe z{ZKwtyTz;TdnOb#d6&t4Ru0fbo7bMgP^{AuAZ}sf-UJmyAPz*;AMqv!bVFD&$ z3Z|hL)A6)_R#}{3J7?@=7C8sK#(?ILr9bKu=93GtP1~oeuQkQO3t^G%CFuS^L0Cri zYzkooxe99#>wtA+tOL?33c?2ZW^6^ie!j2lE0Y=I*N4m$gzbLYiQUNla(E~w_u&9~ z_4ynk`*xD~@871_ z>&yr9-`UHi+w2cokdOcOd)n`SzDn+o8gH6NFlvW{{-&J<31jupgG?2exdsp z142J~&o73DXx9hOyYBRBh0*~J!q(e=2s@7cd-w-q zYcu>d3%k0X4c{L+D12|qov^b&eu+WhhnqhhX8Uap_FnHBe!Rx~yLAIY`R##W|C#>b zr|Oz{ew&ZYMb9X#&lvyGPjFXxE_hCP9u{hL4hz+2D4%FN?{mh0hJ^ZkLqbL6$Z!~k z^hI1!SNzlP5n@k9gi7Cw`}u*F!a{j2!V)aQ3ar8!tV7Gy@u3+_iN6kwS0{!I(l#Si z@ELva<3qLjt7eKirP#TU^%K-JDRoXp`-j|W-|g6m-FRAmw1-e`yKOfAecC6tLh^|1xHezxpKm4i$)ogFMd1(`*YU3;YtZ{e@7q_b z{Tbr}#QpV%W9kXsgXhEre5WX6|6u&qeoaU(k>_2|<2RPt$qbGn-%m4NeLP2fOm?bw zJILdHJBbN6fKxbwbGU#@=tgzI`T!~GxT9{2@$X{)TVy=i*m1)e~_FIdw z@FV5Mt`8WOCR=T{rNjqhj8BuD(z~`AyPs!$0Q38EWJ^&|xaxnd<0ft+ZU2d!w|5=`Y{&11 zC8s#=V%H)61X*p&xrScr-0GU7J0@|#^_{Zc@sXj>af6YMfBt4c7)l?GB8)=YLiOxX z_7$DzLgu9U{kAq98eL11yz}t^>fopGU+hDx?b-?M>y)A}MqcBPwB1ntqW92u7{(Oo z-!Vqx8x0fuHVIXOi^3Fg8Y)dnX_+@N6w{|;24*2W-2KCBdMDX2ZDg23pNCwxb7O}x z&aJCpWSB2)Ar_&+cOaIKmFJ51|3zULeFavb_OA1kR)5NQ74!e6Xe&(Mzb<)v-(w79 zY2ee?cOABUOZ)6keZyw@R&2*k>_$2E;Q(Un`XRCsHAtWdDfIrUwI9eB!TVQQ|MgdeO{)`p}d^2om01DkwKSo5zE{;=@)Pb-AHdxR*v&KzSU3rm;d8h;@q4% zvfeetwL+8St}msI@=jfK%%%&E#|eI^{e9i`ZQMm$jq;DD{eSbFE8ChyzW?vkhsOWx zcSIX2g*3hOkZ%rsR;4DaDW<-f9jSYp3rsFYs2gkQZv zy~(%Q_PX+KyTNvTo;|t3e!bu67gmvLunrrr8C$U(JFy$(*oU5P^a}^bZv8kdXT*}w zD^Y_4(ylXwCVFR$dvnM(y&XCBJ}WJQF88Op{e$?2;m#BJb@__j|0&MXHD(<1;rQaC zwvXc^&Y(?OvK{&LWt8iu?MwAiZl3yQjPsb_{*!S{r6crIk^4W?{r66nZE|1!txq^7 zuL^M=7syI+pjEg;@5WWs-V`5tRm?|>{UDQQIAetLy>bpTn2!D!g&nIVCCGhu`Ga;V>j zqX=7%6o*md7>vUNOhSyI&5!S?pQhN(Ha-)kk=wqYe)`vE!*u!#%))HULGM@lhk4|D zWPfLF?vMJ1h4gN*NQ=lNScVm7nNtu}k!!FH8xUjrQtQ+g^e6pqTj|@8d#G->tB&aB zSz)KN-6%)t>(7RLrc;yCfg}&;`_z8w=Mj?ic8)F zX_fM?@;@2+Q5;8H`=bu^+qM6Zbbbxa^OR>$?Dvy6g#^yv94?^xfcgKie_)#afl0=^ z`K9^!|6=tU#I83E=ij5*Z!I`-hVOep{QoKbJ-@f@AwSr5M~(UE=*sYYEBV0MjXAP< zGyfm8s6#yxJNf@xAIJY+at__NitD(E+qjGSXgk3FPw@XM`TubnKSbjh{y&=0 zf+H#Ze}?~0wmPnbfBjHC1!;9G1|Yxwp!7m|r?zef2GfTkx6Zk*at(a)F684+W)D(1 zc@Bu{4-U6q_Bru~(rf3qZz%nYz8vYvDehab`#9ZwMD;X29ct%4-mhNX2_$!WJ}6^v zwv~o3#T!bZ9hV&}y53DN;E9jkUWrs1tRrEE;XJ{vNf{m=SUq(8au$?3u zHi*^ZM{mCUc>Tw$HGIqEu@BqWfHL!n+D z{U=kl(`2jty1)L3(EXk{0MEzy|DOm4IA?K*DaHlF%N(?Ymw`#v5bAOH4geLSto83v#b z)ogH0O8-RWgZP30`t~RB|Isi(+@dUA$ou3p7V9LOi=c<6XUfn zk{;I%NFCJu;RX{=H=Un`~LZ z|6Qk@VE;6|l`QgZWaM)cu`PR?tehmC1t;mJa0azS%I8q$SEziZv6A zH#~%M_Pc;?WS@IATqVnX@oL!Sy}M4oiQBk~`*?`|ZvJ0EoA(t1P>8`8is2~2N9$kZ z>w^1wT=_z@^?xz{@6YPr?w{)$l26aydM`G;8cJV$m3%FXkv0w!FbPvI4aJy_82dki z+`Peiwt>Qv)*h+3k)$Zc3liZC;$JNTG zE>j%JZSO;JUWxj*B-Gn4foj_|=JMArDhYL{M*>MSAeR3Fjyn|dLk$vWLJDm&Ueo_# zp1-;Loh!`iH{U;FzJHd?*={`gT4+KuT5tp@q|u5^?do^Ld)F?%?oB1(DB1Jb;&7ZS z)5p85uNd@?7l)I!PvH#C;R0G}{2zG<-MEVMA^#!$I=yp7N$9{$`fcRsS!7&y7b=#$ z7Vb*FkB2B|cMli67AmFpqt|V9Z!myfhz5FGn;}u|USY7b7SCiTIUMOxC83BMg_Lyz zI@qqx`vqZ)?QzJjJE^|tVyiN!K2scOQ0rONT`$(Y_gt7 zd7|H!%XSsa*i|qCGcbdzK!Ph!j06*y3lqDVl0X6pB#=M?2_%p}0tph3Kr#xhGBxHk z1JlqA-7pO^rj0$v_Bnpww|yRvKmrLQa03Y>kl+|3YA(c}3%Z~S)%mPL!&G}6xHaXsE_=^RMvKc|aApZ2(arZ##S|JQf? zU$3=6h;xH-Xji9n>{kB2QvT7UTz6M$pXRcEXRuSZRff6Jn~%oo%1}ve^FG$wGwCaO z>WDIrwjpiuQ@Z}pBJug}c%<0RuZ(R855LgHUnmO8=qs=aYp@O*unAkR4LcCm740Gu z>XasVXm$>_T7~KyYlb;29Z7{|7Ya?g!~_p ze`$5d>v?n*%fEW8`?>t17eiG)3d3{anujlaKiqx}>aw{1_X%moJ;Y9tXVHM{748$?e@Yv; z#P~aYWB-5Qr16Io+cIxYj%IVZ<`A0MmMzzd!y|fwbq0<2j{X$S@e=joI`u*FxjIrCA8udoP9una5Er#i^v> z4(r6VD5K3+i^B%`CbS8+PSFOF$=T-4h}(u8*o7MH@ca4AyM=4j6MM=0{Jv`?VITbv zj^G#)#{Qa4@xNbX+pA;p-{rojeMcLg_6f(F!Wo=J1DcRR2i|>u{MYjz&G${~-(-+I zrJsZLBmD1+-i<;2v)>>8N_mt{KXOHE9HAOT8GL528=!d5qWY57k-UU&qJ4{FAWncu9CE zeq`~VhS%};sUL;6`3%Fuk+=nVWD&uV=0!S zXRc$|lPl?iW7*$q@zwOT$gxec!WrY{X)sg3cYO2gT+1>qpQUfuUged31eMd937dkB_I42@_yTgFE)QQJJp95%l9V`D?o zm|0uFIRD?+(D8aaAM1F2)GciF&7s@3(7JYS35TV56vuHAr%{JEPw$r#UH5HisKd9@ z{#E$P;=c&z{FcD}CjI5P`dwvz5%yL77vZlCeQEsf&qC5~?f9FHIpN2%|2!P7{IhUm z=AVV1O#Acj)0?A0?Lu{j_UC8ge?R=(^FF@L_}?nyf8MWO9P+-(@>)STM{172YnN7;JI&a zOt=1TviIke}~Fqs@)UJ#~|(@~CD==2=skn>Q11?cfS;~2wY zdVc?p4f+ShSq|_e{D%Kzitm4|vIpyJYxpjkONXV-X{=WN?=B9@=`F9V|2tb8R?=6a zO}o%~Q=3gDhl;~mah+>E3G2y?h<5d6ax0>pJuq6kkL|)ck=35&kVXc#Jd=gutFQ+( zsKr4XM#9*A-apXIKk!tW>zou?>1~gU?>DjUaefK=9$jnL_voSbqHnkL2aDXtSMFn- z>(VpJ%nu-Q^UeP&`#2nRo#Qx((`a17{?-oH(OZ^jn{kewK%2fzD^f_JW}Et4pDo&- zNx!$F_N4Zg-PJ|kAB@*t@|{3>8W(U0S8xr9b=vRp1Q4`(l>9z4aeNV z9o)kMJj7!>LEjqfKL*VA8NBRuy085sv&ivpw0HAAN%Q<(?LWG9zw=G^Y|;KB?!Wa+ zIxmp5*Z(WBXO;J7?nmJbeQ>w3x#J^=dQXw_Ze{U)|H@I~;{N+3Pf`tM!}D_)ZWR=a38>F~>?jIRAa%edZxuRtM>irS*?9;uF$L zq8(l6LmC%w3Hk4OXN)ryC`V+gzI)pd@9rt@@Cp6j9cAH)=n-zO+AK96>E;5<6fRc!2gwD||d_Iu}=e^6@v z!Fcyk=03=@aORQu2T#pEAnT8P5k|Sr{^|Pniwi;tz44HF7<<16W9Ti8X~sDE1hhG? zb;vgaUiy{*CW-5u`FWU3PDPyeFr6$%oCh%=kAs*cJO|lb`sLO7>1W*s5;x`VuKW#| ze}Lo)^AFI9Hq3KQ1r}f-7Go)vV#-4yH~s&2%fe=Q%S-=1w$isF|2<9beCm<^ANl>~*Qjswf4(*T zQRSFDs6lPyNntoh9yqA%J}C-^=|^!KCz05v|9|4$oW7O@?PA7Q-);T>o7zd7c1#`4 z#bc2~JG#&p*}?ya0SqF2TUDhF%#u0vMtisZzckMqU+hHJ3jRm*pcg|cjlUoH`z(P=YZ??9e`; z87)YvqvF1~`F(WB{Qmxr*#A$}V`SG`^%nVlfB(Mw?Ef9w-(CN_{oi@_<1o&3CSVdK zqsRGi{Cp~Xus|K?nCbL#sL0IaT#!bdut3L_L=`DNN|Bi|K|74TeWZ}4mFm+5{ zgk0&E&L(!)V{JgWa-}U^Ep9Ek&$@TzJ=*2#g*PJ3y~!bOr;~}Bp0E7n=P9V~liNJs zGurALi?1xqSf68#PK5`3wd;BA7pRfszorKAl zif@&tSf*;w=Na}R|NQ+2Y0v$`^QX5f>;E+V;rV+Xy1fti{+IKHHtT;pWB21z(OoW*4Y12a%8Li$8_y?(??;s@czx_1=vS$CB3nlUlptAaZO|MijTrt z`g+7Y~oy65u33U+p!Z>*n_^5@qNb* zIA@Sd8#B&Sv(d>M`@X%2{V)E!-#f`J$H%?>^Yj17`2D{c>D1yN4x^{iF~y&Tqx3=c z{J=K$B>g0ED~!KyVgGM%Ei#tf`2D~9I1s&Y_ov~s<)`5W8OLhV z@)yTy<9GjViM!;T9`qarJcIg8pN2c)?%@F*qEURqu1~{bdVc+Ybf3_lq0RlZB851Y zydYnp z@2d74v5g)?nw~*co0&uVXm$X?fk zha&pmb!lT1y#zUW78#^5MqK@KeF$;_8jpM&CXtg7_t0%VBwv^+JRPm{HYAZk-v7Sg z*X?h$_)X%s$v@e4=HpQAoLQKId8ohwEW~2;y_A1_l>zd1_7C3spP2us-S2h(OQrFC zzTk3toFloCT#dEJ&6nTV@=M0?iudCg<2lNf3W-AJ6IFn!L{aoiHma|H~T$) z`!CMz+`ssnaNwc!1Hy4`XMX;`j*r7u$Bf+LpCNam3VTq4S{#hB&O1!zrtS_$$>GhP zgyZDUPXFJ{kHSfMzWuMM4yWlo?E5&zRY#9wPXpShbMypq%IXLIxiP4wV(*}G7034D z7=N5w9QVa;(`L1t)y6jUrNTEMe%tk);~pT^@ej$z7!*ISOxqwX&b`RZP@m7&U-8?> zy^q5)@&#Vu4Z?pbewOlA{&g5K)?XwX_eL8P<<1aF$RYjB`l|(D41FB-ZQmUxkc}73 zJrh5P-a01L7Inf*!nLRd^+iay8mPy8kfO!+1Z z;&;~n?f52i;5<4PeG|IA@*gexCiGN(6ME;0b6?A)vl6SZ7Ck52WA)czJ$u`-e z)K?vDkhgFL_wWD@@fc6=3?mo54ll@8c!Rk9H(b)^z$ld9{rcZA^g-opKzpB^s*V@_ zu>SWYJ01D*d$I7J$}fF_<0hd&8E?L>{PHVI7T))O@0uRx4JA92Ukts{SMl5M+Al&m zIr5?~%p&*OXZL{hmcx6#4s(QuY6`+UvH}Y*^6(GCLb7v5L0C*K#d54f+*_r413QpD zsEo(G!Pe5(BR5gG_j`s+Pbml+#cjq`Y{yPiVGl+wedA%f_M5LmEjj#5xKus0OMjy} zgoEM^<0$H1OE-)Pan0{Z;nQe5rA|r|hdO$k``g^F+#gf!o7k3QtKUDg|Js)pBxh(F zR%jzuY5$gKH=P^D&CZd>Q9tC{hNp_c*@;D=?m54tbNUh6=)ud{sQcO~GK<`E?cZGO zf2H=H>@3y(kJq-BY5&RI(bm45-5NUC(s}=z{RMl@+Dp!LlIZT>H+aNvz}Md{+=VPX z?g5m3$&XL}``1J^V%yGe z96wEM@y)o+;n?oY;b_@^8NQveE&S!9|0?uJJB|IzHirWzc7@tgTf#p4Rl~0E*ZsS~ z1;1Uw-|W~Pe!O>2I6QwxID((J)=!(N!xg_>!_T(t4nOyNjt^CZ6VLo<3p42zQm5vp{;C8 zX#H+18|~dVV5fC(=bujs9T)kE_=~!fsmIdk9-6=xGci1&Kf?>W!iV{pZwtd4;rj8# zAzYU4gX~9r)lu}uX~uu%7Kajg%N9PA4aH#$eH_}>C_~~>NbV>O6U0qIZBa>>Oisnn zR(32o{N%q5<>V~P!ARocFpsRj0xU!m|8`<8A5F@2PD%Hybo=>e8eFHzo|^3M#g1Ew z^9$7v>s@CfHe)M(IKN@LaNlg>ml)8`h~H97vkfxc`Yjpu z0olHV{f`gxpXTcup=Yc<2=>=`?n_HUm9+L?kcv zQOJJB7QgMDc$!M#z2PIOgO88Rw%SL;? zXfAV~%XmyVb2eeY2nNYZ2@vCFwyx9TdcFJ;3#G9bOHr@BSxz=Il(Hdw=kK05b^Q0M#WjDYE<)0rfs}6mwxaF6 z`7DdHN6WNFd;s;{x3%&7uYC6w>*@LZe}Z;+rgnIm_pws>B3scm-kt+%*Z}+dzSr;g z?ee!#8k?~d+p!Z>*n=AMZDQYV(?=i&tMv)UjBu9B-RJ*rU<0K12gpwSr7n8+8STso z{(*JkM$4mg4&pG5qNh|D#c_Ime_eL+N&0Ez+;3KS&LHluTPN-u5=bJBJGPVY+rM38 zeqK1cBKifQ9o;7`jSGnL58}R%L&wekp9k~*T{B{jDa0_>E4-fDVk1@h8Jo4(Z zuz$V&S&{nfgtq>g`fV4R!tc+}Lw`ZOLfm_LV2yXy3k7+=TxasKo~?J}~Z+G%5s?YFi6`UL06PRDia zHa?HIKTq!lWWQoMJ+6t1^WV$qv(T2|e@6<*f{#t`^!{!0{%z8K zUg5fT{J%S-k2#Lt|3JCtkIL^4Fkio6hWj%wAzSAD?#llO{*N=RLH=|3pC^q9EWko6 z#!@WDO7!he{xE<+eAxf>wr8v?=JVen|L8oT|KmE{EBOD=i`CMom&di_dNk@wt@OM% z(i`Z_inzwYC(UEEGoVGn9hi-S0fqd1O}IE^2+Uv6&scTWlz00UHjtxSw*-wdJ_=*V31~UXKf(8x zYGV8M^G}@MpFpiKgzVz7aA2eH|0mY*@?lMKe5?m1lOv}K!&GuQ$}tObFb@@2fQ49$ zrC5%YSdEdc*nckzLwmd*!tonG>&cDSjIHR@AKy;yL>2ZR&h6}eS>O*+?$3HR?`jVo zP6`Kwb18a*dZ5F*${t9JH-0$L_~8uWhsdw*pJx1Urtw2^oO8Z?{e?04FJ$VAP%C^G z`+cA2fVGr;7mcrD0E0*)gDi4rUuXPMI_ISq_qy+zV*Ju|;u`f{)Svx49F^X2oWyA~ zHhgZ~>I+PkkQF(GzHU>fRn1<9zO3Uqf8yDc>0&yU>&QBJ`1Iba(s4y59zI zLHH7~yN&xJjSLd*scD64tdjpN(%UUS=zxpNmoCc>^N4VJkW&T)}o}1~vpY0t$k>f@o z&b`R8^-Jg@33DIFamY=d&Bp&COrQ_V`#ek{Cu1t6qa3qPf89JZvO(W+9$A4#aSOMIU^LXEaS89Nja_!57y%ULPC@_tjgW4EA?Efvd2gtL}!$INUNxpkd4zUT2#;`HKPS5H%{UlDK zmJgukvTNKg2zA1PH{IKH_B%a+T!(kys{j6?dnN1d7Ke@ElW0d58jH2RW#;bFTeQ8+ z?DjPM0^&FATG{O>ZEupiB<_d)qbtJKa09oH-~aoPc7c8m5AYC=@dWw#eeC>Zv>>VP zkSf;3>p!%S&m8jtukZ%pf6y*q6iU!HQ~%>S|JY^zG5x(X{lorWXY~J;m>;;x{J<6F zYH9mF>`y#R|3g2ZQyV)*I^!?_lMv_dcOUyWOr{T#aSs1f`gG*p@@*i4wByRf)mNK` zK+ZwqMK&&3fri_j-Bs;3xj;Cs|9k&k<)q_M{o2~^_~+34+WbEx=_$P5|G!2bZSPzC zn~NbVlt$e1?7&VwC;AKJ{6D(>@fceP|I+^dBgUMUN^3b*BCd;BO|Hdy3_tiJY$T)q zYjX@6kJ=i;-r4PBod33ytisRCgWE&CpZ``ve?R}NmVOYwJ^$^nxT83Z`nUSoWW!VI z|JbUh>2)}VW_3ZVBa(=9M64Uy@Wc593CAVTj_k^k&`0jSz`mZUPM}}FC0xNZ+&~2a+2DRw3#=PGegn?DAvqie* z9%QUV$eMeQ+hG2|RqNJq{w`kwy6D~Laa{cV-$7}_b?iMj9Mjbo%s)@#LY?EKS0+8_ zWRY{uQO6v|fZqm3`+h*`vv5*4KmX^7@4C<%#WkRg-m*;IaE_ip8$GT;NFlC4NbHb* zG@}K{Gv2>_-aoW;tE->z**mWtUFbs^7jP+tH+cW>;rmnc46?|fUESZ&;Qc%6{X-YM zJLUa5=KVwG>(X#VI@fRmx6o6pozynop$~50H_|TOqd!1yi~g>-%r5pX9*V0s2mdko z1dU&rPfWf*!wT){GUZmg`%3r?^6TnmD#z-L6!tz17Inq%*;#{GrUJR9ZpZ$4?Aqmp zp@cpL<1hhn-|tD}uyym3$*Gu*a&#W!yC=K*_5Z62!yNiNR3PqoHHi27kYv@V(FRT| zQvO%DPqb`uU)AobL;Zp{zbTIYFL3@sEXGnS$4ab5e*dR!p8qE8f2lTgqUVn+Jy-1c z-{k+phwp#v^8Bkke=@GW8mdxe=UDsS#}8nh|5|CT$3|?%R%}NV_MiqM53T(-H=veY ze}>&+Y-le(SlkP;_O!WMJB zoc7dNZ9Kj0{>NdbG!Ek^j^iXwqYmeg-~V?P`@fo>o=k6JUteZlUuR#F?b7Lx_IYhx z{60{Z_B6JwJ?pf;+U3q^+Cb?f(T*{?ysdo{B`-Ct^A=4HCOdD z@hvy`1LtR}^hxoM{uuw_{M!}EAI>B074zHkZ{s%!pGYUa{{LP0xn2f2Ij&jVx9Lz%n7GNP3V=0y+yWReHp&|@jGETkFSlQuEKJ0@M-^md^ zuzd^tK1QFy@xyoiv#@`8WjJ7rZLRRgIqOEET<}rYNN&bfY{yPip|+wT>>+ESj)NFB z9(kCIH<5w6ikR`Su6TNk|2J1snXFoav=9o)kM)UWU^lDRwoDm*5ipmDeI zze7DtZ<#M|vp)(i=&z9f{-1XwHC|qEbmr7>tay6(N$K?P)4S8d@qIJHiMi$BXP3*v z&%0-blLfQFsqwSI-=3HoelcTCIK6F7ICE-l_`Aw^;qT|p3oE7BbNkECefi69iQakh z%kYK_|C9F-xw~J6tlu+!Ph*fi@ZFbTl;3{5Zfamv%a4;)j|<(mjf-Kj-=^Z*DbvDV&X^Xa)622{@{Dj`Z+WPlI3w)CUlq&@ zfBkx9nB}*5_?v>+;m3#Ogu};Xg(GFN!%r5?4L_YYH|&3GyqHh+XKUt%pKq8Kj=!7} zPSF4MY(@CRqRLR=+y&TMt)9l|uPX9$#uDHCC^WtOkv`UsJOl4p*^k&izFlAXBlgda z!b0aP#!@WDO033OtVf^Y`p4V<0E5%)e^6;Zg14Ue%P&Lj>6f9s+WrS$+5Z5Y=tB20 z`yV*2ckYiu{oabOQ97Hk72DCcuOc*@s0cghE%fI8icm%0gSJBz`X&{|PAWnYwO9Dv z>=jVsw_5brL*O8J7~S-NRAm^n2gFg~Jl?*?(Y){Rb-T zKQPVy12gSEK(?)sZmII2j~n|2C!K#9bx6dt`q=);Yrp=;z(nJ3Nb9R)kadlCopxz; z%&iRP(OFy>x?HRKEZZM(|Gj$a9h1_^ukTZ@w9^|?@{cZh%X9hnZuQaAXxqg1cvt>! zI&Y{VToBhe)NetM-RPsgZ6j6 z_Sc;H=2O4k|F@t0kJh{F|J$F0$*wUKbKJ*tvK+Hw81vAl57DnLF1J1&H5g3UH?|$?Px92|9GjNF-1RPy!`jG|MB7b zUzO_SY5v=p+G%p9bAISwsS@6U8f3RySGd>wpdTge!=9ulDVhcXB{KE$`R5tBhUN2BX8XX-DT*p1~4% zC%e|jJJ} zvae6Dr_Zpj57}3a4d1_yeSRd`=iX1xErTpQhrMUChgk1gv5&ox+>EW*j{R|LN`G-D zy$X9!gW5=Z_NUnJ^usuc zL^~SK@PD9--g1m>o-vPso<`eq_WL3B`(Cy>c|qJIT){Q`aD45Ca4kQ~E%LzQxR+m1 zxI@2(2Y85tccjTX(@eHZ^WRkdX8C{Ym?wCK7kGs?u;{d=2z<1m9|L!l0py3 zhq~lE*(vQVdi*v_kL&fm)+R`!1Y^*7%-UV@{rwr~iI|a8`e!c0yrT zOkaxSSc%nGi}l!u{c(=Hv9-dV~{wd{Y7(=b_&VJ<^xSj^>zw0h^-_ZvU z_mAzO_f+z=OM5z^4*gtJ!~V<^Qt$UzdNf>%RPxJ;J?Y=Tl?*(%Fjb*omHE$Bg|rRMGS6ALm<_ zPOrgluYar+SMPs1NFGMxSN1a?YrXdktK_R%z81M3_0UQ2t?T4VI7z1bo;bz+z39C_ z3zA4Fzpcu38}=Hjufb{Oe*4-Sfuq*n``_z?&!O+K@%8I$Y;y3vegQpWj6eHQKjF-; z$LEdFpGPOU5WoG=gI;tl(KnDz67A?hkGO91(FgZQ8)^Cluvv-9G!naj$*P>4D>hZ-nrWe2ga;nP>fM zlrOZI)0Os^KNt@?#b1RxVArmwrcM#Qb=N|YfMKu zW?>HIp#lq#cxr!tWAn|kjejG*Fa8;0%*L+Uj(i@99J3IMu@uX(605NmeND!{F|f`! z`wHXgJB+I%yURHH7US}7jelP^{(af_cgpy8zwvMM(0h&Zchc8OXCpS_xaZeX<``@h z9vm(0`JadF^tgY2uG-i-^85dfHFmBKQUCVyck|FznE$uN@7jf(j&GnhJGN!n=kLb9 z)#t6!YD3a^e}4b}b;kd5bvTC}aotGJ2bakISMH16j+}5- zI76maNKagSlYI`zG#cp_$V+ITHy6uSsd)^-*U&25CY&5EpJZZ+J~*1uf+SLCMH_B7 z=N9fDyWThe`2dgc1bv5;e+($=gRxBZE1SqF7df;)G6t{yI8Wxk??>-ePxRbY{xj;9 zCgq2%@@+;Ry`eBXlVp#lrA5Z_Ldhq3w!W!9w#4?ipnOUc-eSWb@U!`E)qXVO2&ufJbd99D~8i}l!u z&De_V*oi8PwClT^Ge_{==V6cV@EHDRvK9w%7}2MElst};IF0=J18q+oJ&s8X;2b@H z+*@rZ;uv}SR(#(6*1p7V+_fB2XLPGOZmL7Bs!J}Ki{_jp+R=r)KcU0_g+AfF4eW0W zV319n-o*Yk29PClW!nEk>MopToBz)IKgZN-=hD)-fH)8B64|)o-Tc2R^!NMkLxkq+?W|4bD4sj2l#BOyb-meqP zxBs5S`*ng3o&Ok5@C+~T3U3f@%l{GW{|W6s1{>rh6Z<7U@c+J+e{KEwyV`nmd9K}f z|NW!-0&BaZGYTabgT{&S@41Ylw~TggmBw`Glh9V`-jG5Plf_NNbd=+V_Qx#YTJK=o zx8T6z5cV(qEX)&EfdyEI#2M|serWUcU$_7C6g|IBkFvJdF-x%=E0L`z39HGqSdYG| z%0C9~DxccZ^lRl4*;C3V+86y`e%?Cmzi{_5<(uqvT<0Rbf9Y(-R%}PlGVMQh(gzPo zdv9T=qVGXYI4hhX)5p{w;_3_Z!O4SY9BpiXJcp zR+DSd^PRZ{L8 zoYrU`_;~j55j3Xc9|y&^Ja=#3S-&vd_`(E!vHS9WQ~n>xzxczB>3pgUAde%?{X9vY zM%?E!KQ6t^`z(A8S$b}r|32EF^n^M5P3HACPf`AnL<(__&o(5SlSDhZ@XN7$=BRVu zXW6xVhxp%-_YT7@o%#8t+{vf?)&^?&h2$<+`r<-m;WU6 zNjr@TXgte*DE<;XzyGNFxI({%Ha_sy8SW8r4#D3|vxlQ+ar(R83ungLZ*b8c8E^T$ zaB|Hbg;Oj3DEw^4AB3Op{ljp4)gOctPyZnNze;k0^gxa!mJK>W~2`vsD5{P=IxUB6Ht4i#yS z4;F{vd;eJY*l$np3?u5Ek;CQ$Y{wgZ2X#+OXYUmKjQM`+dRiFQKF0M6uN-ZNa@!J@T!vsviWF*)aO^=KZ46!wkr04hR zy=i;^?duOy7#c zW$piM*FgN%Z}P3Q%lJme^P7}fpXr$G*l+Fifl2)F^eXH@4L*E-=aBwC(x;RwWa&Ay zD`y?*+Vf=Rcl>YaK&Sa%lff`;YT4we{NGuKU{m zZtZ`E_8-}6h2ffXZr~2?;Q=0E zwy0FcjQ#h8se)&Ptuy~!VOzn!D@=_24~0#q{zGAN!HdF{vVULLRq(j5d(*!y%wBp^ zSc~?8CxsoWMhed_8!7DE@&6Q#limbO!emUvbd+Nj^85eJDdJb;-+%mZm?Lf;DzE@G zbG^Tn-d}RD@KP*Cv<*fs@PDaSR|@Z!#{+xb7Oti@s$=6E%C+>CZT?%Vr*A}lUiBLF z=L&h-pngB{ePPpy?+csHd|%i?COf__Ol7_=Y;F3!unh~Hvl&~l9XnBlJ*YvSXV*XT zSA_%QVCAn0(_}_COXjZss<2%e9XP-D`@&9iq5IJHg*{t-RoGkhtHN4o3{8Ghc#s^f zdQ*6qJc{EuiPNaVIV6y8f4x`VVx;74VdpCMEWRDt1^Q=XPql08{=Tq}J~;FH!U5#x z=Ow+9rQa83i@z_-jQ_qcjrtSr*>M+e30KgV`Bh;<|E~(K(OaJWs<0V1=(o@&Ppy}K zRhYW{tHLDii0gd*ec?Uw0eU>ohvZ{)(+3u*7nXT{g`Xk+T{z+NDrJ~VxVNUco`0q1 zzeHZX^8Crx9iBg4IOi4KAl&s1V-!j-27OD+KatkJI@cl7(#-67*T;*#=5}>|$944& z*?C|6Lv{Iw}lOuOmES?#JRWA=~?6BZNhQxZHkP0 zc-AcOey;L=Ag=!_cU%M6h*|Xf`oD2(@EQ8y)6_5Om=FF1!}6w8sQ zRzIm%n$dzJQcdcoW9lcg?^VAXQNQd{*40;Chm`G%`b8bnTdn?bAN}fw0djDiI)Tgx zXUW`a?gDj2XQ?veTB{M;&b8!v#5I5e%E+Mdvr%|6;@ZF5I{lno`Z@Hi;`Fr zc|XW@bfFJvT)-uC9xDh}$oK2ZuF-Gc76$y+gWiYqHt)w<_47+*;XC%t4DH`4?F91v z?-Sa;Gul`(?h|_4xp#065AYC=@dVG1pZ{0Pw$_#pl4&-7hRlz@SF-K|xR&y$^- zjK5bKf9EUcL7e|LJf=9jkj~JvqVS6R(7)DZzYz}qyJz(4`3FVx5o;Vrkq7u=_v;VW zyVnxoF&KvlXw;@QU=qD$x4i9OC)1}Q?it+r+8l%D)*9%qB#y~Hnvq}s*DwDK@}H7_ za=LTMF$;4r4;5H|h3MO){CTGNezs?v-`7t#zt6w6xc$ETOXs}wI?4R_N0a{1sdwWa zaZjYh(pie-Sc#r#`cSjYEuhD-j{(Q5rLRYB8(;pWcVi#v-R2sItC!c!mjIE zj+2PCf8RxA>#j0JM%zCvF53Rt=gOvX7JUG5@BO$&FRo9IK7d|CA3)6#Yw+-`vKf5= z*?achCVOyN`uXoRK9u$$*FajjnV2>``U2|JLv@ZjhXj&{K7eTZx6@lLt2@v|??W3s z`T*h{qtOSD7Iy)cFxC4uV*Kt3{TgmynBU|UIdr%n+#&Db0UqNC^7}vUQvX%+A1pA> z_8EUbRcUzUw- zWHQ>;WPYDK@}YbG*0}!ON1yr7*Joj+G}dGPN#k+WjOO?MTg1*@#?B_w$k4N7ZnyaX z>)5}q*}v@B&Ia~>hW$&%b@sg{J`EeCH6o9j$*tIqov6YdWIg{HvK9w%7{lgK93_YB zIZ^+}-hlLzIE^|qifeeucR`P93YyvD33?K7O?~V9cRr3J`QiI}E94&^_BYtU|ABVr zbfFJv{LsI4A;#Zi|6>4yuA3fW|09bW+Q-U2&Z84u&*dLI=tZY{yCR+Z{NFzFQRwlv z=p(s7AG|GX+@jw>ZmzVEx$IiFC$9bs`;vT!#%}xmkx$UzSx4VVoWt-;_yt;byvtYg zoy0i|AMAg9-IgQD{wZ|;S`VoM5Whe7%6V@P_un?aT~mZnD8UE&-#LTFw2S@PMZ`XM z4(+em|2VH+=~SC+Lvsh#zE z#_#c4+vVco9(c3Jd5GV>tsobmVU;>ki~B#Wl;&!z#mFt? zU3<8mzA=W`|Jf(}|4)3sLU?!*e-s(*q3tnj55~Cv@J@Oa_Misu_aCmMzu$lOApJ0M zcir1f_c-43L;U9J9PvkS94FCeu3m%rf2ZmB{r{gAhdTN>v}udt9Qu@FlBijs|F=f{ zw}CH0pCRFx_v`PI^!)n2mFn-O`Ul$HG~&KJ*_-VD>-q-ag!>NhZy-PZuMd&#*ES=&g8h&7N810V+W)uO ze|@05f8?_E|DyKaaS8uLlmDc7G&>#1iR^N;(%VqG-v70S?;)YTqYawinv*aYQ{(j= z(~kk?432$Qr)TCnN8dK~Z`;-X9bf7HFN(ef{eQB1nf^c7>$vICD90?s@Al3i-+y0Y z9=!q!kpDi1`XGHq+1sZKIyOhfHNFe|z8DS4^-{9_wmoUdm1w26-GA4oNRiD?mB*X> z^)Kuz;`hXC%-dV*nDy9*&De_V*oi9iWz2utWd74O^PkAHb~r<3 zg>z)PzHWYhwd>x`CiY>!Ie-o3=$thN5H%U=J4SndCtABGy&7a!n-4k8dj8YCzrM9B zTvGQP6n{AWmYx)jlE-lpov-y%$U2-u0&#DLxW{Yuo-uLHeqgu%ReU@0?Qf;>hqPmc zw(Ap%A0EY5Mvh#Q2l4_g#dzyIuaHAG3c@uqKmX#zq!7RVcSE>d-FAz-gL`;@M(_{?|b9>t!P90*e}d4`y!m@KkdBwg}i(bx-Wj={r-ag&szM;zTqx@;I?PcWPU)u z`2on%b7!4njeht)l`)J$33_fyqrn<<`d~cwrTzEm6OePA?0jidO3QJR#7)LjOh-9} z)_)OZk;7e+!W?oQDliht!UA$37Go*u=YAfRlPj?rYtd+2y#r~5r{|%X6e>zt^FO8k3!XDJ176)+{ebbcxnbMyxeWYhdAKClf|J&aG zWy-%}&O4@)?5gx`l08$Df7d!HjpI0p_zkJkWWy=tnqEgghXk6H^%i9}`BJ<2_ML6e z7Q=pPM;BuMw~tKY0xlu%FZ6zWTjo6$zJ?n(&=vRpF3S7=?g-z*1B^U251xFCCm4QX z4m=s({}<#dyg~TCyff(BU~Hc(!5ECg`}IE)=+WjMz$E%)Vyp9_;-81_8I3# zdp_uz==l_TKIrwV`pEqE)wX%Qn>^nQo-f|7uiUrxlQ31f(@~Cb%FHZs4(1^-oBv|I zdhe>|hvZ$)53Tez#65e4YCaDYjvY?>7J)q$7DU~9lO0nO7Soqvq{MqluEc7rjd43a zH~-E0zt{$Ur7lxvt{4A){m({v%R1La{`)@_eD*`$t+(DWB$3$120X(4C0kBu|FmUs zU);Pig-ejDL^TzE9M?BftMovG)C@_7CULiQoP{ zVB8b1Ryqf97)Q}Fn*CeA_dp*UFKwKppGI!CwsnU#wpd$>I&njFMd2JdyjFjt)U%lD zeXHc#5SK(dYS*(Fw<;6QjIkZn4u~5$Y5)HtpM=^`+Sb5sJ8Vva_{Kx}4}0Z>-f}|U z02k<&(8hPr%6E|3r~iN};yS8sC$iq-{P+JZ|0n!^jw_PJ zD3qZ7tbP8-2C@<3=s%wFkA)M+=2Gu(vG=#k`%AV8w~;?LruMT<#?}^%4ad*?QTXYO zKMFrtHQxS;|F-bxntv?(`wbt5znfbe&Mf+qaC*z1gkP+hXpHXjaB7WtPmAnrvp$4L z(jQ*>WtdFXcYG9ndE0)7-JgWIL*AW&d7*vum-?1}5zZg^3u6cWDs&Z0wYKmtLUvY3 zXuA66A#w4~LyL84%|rj1|KdMaNBvo7d;Mi-EvN`*C-Pm!bhmx#`)biH{Jn6_{)+#w z@ApIFso(Rzo0q@PchE-v+rqy@?d*Rn+&6w)*gyY|!-4C69BS)IL%H++YTB6a*QI|L z4xjnMaHMon_?wAegdb}QZ>ft$LSgvHwh7^Z7ooK6r@zlwl*#tB~KnY=u7D8hyEA`u{UZ!yfT9sKr4v zuCtC{O=&nxZ&_k&xSG99KZ&+e_I^8N4>&TpqVzZW|1-^{c1$NTMBG=tE+W@%g>hA2`1SNu;J(f8ZKzWcvo|54KrUJgYniMIIe##{{#k$U0;0Wxw;iO)Nf!%yH4$;f^dPngvNc|6Y*E*Er;09 z!q@0G&~}9X04XGKOWYmY!vj3TV?4n#yg+RG;+&1>+k7Sb24P5D(53!OY5%?N&G?P> zzf}Lb zzxVZj^$mLc9_`1;(wU0sC`XUBzWW55njYue#l7$6(B~ol{ks`_aENPwve$}2y)ste zw*_dt$Q~dUqv58u=dF4}8@*I`Ia;^z+iQ>G+Mg6XA)O{P&)5G)lAc2A8U6ngA*^)H zYOKY2Y{a)s%8ch$gT7PppVI$t(Er~i|GVVBTK>tLaJ&3=yp(@*UY37(@0RbLZuw^q z3{~lav#Vo2d$@v~E&Z*?jw=e=$=c*cp^7|E#XnzC9QM#_Fk-Ksk;Uxq=h|@LgE)+% z$bW-($hs(c+!JnKi#Zwe+Pb2Uo3EU&QAe{8VqfBPj6Yly>Y@zu!Z|X5B-+vJ{fqBl zqF*`a@b10kzZvrG@eie5{@Sm$iJys&r=x*iiNC8?xQp!G#dowW`lU)kAKAYu`lU+w z{Y&+IN<-#lX^7u{-RU}A=tCNf<|4;7-~v6aU2C3Uojv^u;x_?WCu*a{YOAzc*Tmhx zE!@Em{rC5TAK)SK`)~3WKBhmxGrT~e;n&~y%do$%YE$oOThU5yyQm+bAN0y`ZxH@( zeLjpr3C6%$?9hMP|BpfMc;KsMoz-kDdiOTt4`lC4@5Eg5 z?4&aRlQ0=QYxF->m^)1$JgW^~W*mTCj$D)ehqw%x?zZobxH%YkRvhM$6cOK`Q8!2 zEP9;dGLNi?ziT(LEA72-v>+@H9=3^UAQ6z8-B8^{d8y8B&f(VxzduvtNeI8KLk zgDgFVG%`r6`Z6?a`qCVQFGCBNtd{l`*XXzY9W~CW#X%g#Q5?rfoJQX&>)+Q||4t5W zlkP6*R@<|I%n7$&wf-IF?^^$kE_ye59MikwNBMWBe#mt?v?*wO`lC=M-7I^v;dxw_ z@4is`&{(YaR(czfNTKt7MM#Lx&o3}ulce`p7vKG|BDB*7t(_m}u$G+Ohuj+Xw!%F+ zCXK`j`A0tg8>EXAK79YH&O7D23%G8;yW`lFW%kq*E*n=8m)n9R6kdf10hJ(WS z{eQN2h8xraTB^Z2LzZ?(TGw7-b^ z-?U4oE2f1sdS+;UwQ(2dgFCdp+w?!^SCC_iXSL0lHQL_|`XA!r{x>(sTWECb9r7L; zHofb+v}h+E2tP#Yc=@`ipD|57%gm=P_3X4C@%?-3_a}IU+I##>?r~LBdE@_w4X7BHl$4m5nv|H7Ry5*BWl#2KWsf%5L=!#P~WNT#tBZwkS=gz_5@J(Z>0P~`v~P9ItkackMd7j z)r0JXKH8upk2iDRti7|Aqt1b=XVX7Ty#18FUfSU2FuwpU+FmE9YkTev=KdR~2aqq} zXNzUMoq4EK#;>({`jfb&U1mN--LEmlT)}zDI{o2hq*@oKXM(*#m_wj}RLFpA7$>ds zkm^3XBBXjYP#LlkOkja}Xohy^nxapETteR1k#6WckDkGpx4eegg&vR-j$wWS28!vM zfX7N-1?t1-|3D+}t)7+Bj9Jw!ZjGj%qK?uGkw$)eAEeD7*Pn#`L(Ca;qjSHUGX_Z` z2ld$8`Tyrhn`zPr`Y{hcJN_Ne3F=vi-GSe&ZT@in{~6-@0M5cW_yj(Oi!gbbGoX;_ z`2R9;+``!i{z39BX8D-!5Y)BL1xSBb3`^jL|9>sT+|4L>)ymjCmiK~3u|E#2dw4I#Ld|i!Cu96B z9cQlL{xm!b&p~ez_s9JO%&ubYuU{lDVSX6~rnx`w;o+Wccm=mZpn+5v_YaT^Rt!dqkjDUnhD3^Q`Kdtub@y9#>?K?>S-K0NsL3af2M!dbqKCY|PF%~4AG}gw||9`xO*~ULf zthgV;+*Hi`kDfCEF%Lr9H1j{~6KT;h{{ts+>oQXIw^Od}@sZv=-f|kZGcb34$bQBJ z=mC6y{VeovWLyC5-IPIS&qqhdfNqePItaQ?QFfsh)PHo@FEPHDrVI}e&vC9BK>?lpe?xF1NXspcnEgEqcBOIr~d^X8Pjm4Cid|qj1Q4d!?W-lyZ|r3%b@Ca zzJlzb&!_(1`4DCeq=KqL?$Yvo82LUTh&PJw1KHrq8Z`e#X`)?hq&;j==b%CT_m8?) zd=1yCZQ~E~R~gu|ArFecv6XY-jm-Tcoy>R6?XP3*^N0O)>c9Wge*~-l$nEr^U8j%H z-ND*-3v1xW+I;#L*{n~jrTo(WDMxAPQqz810TR-y|0`_ zj@{ua=aA$4)A1AJ=Wr3eg4z)NzZd>pA7XBP4>#K|&rhG@Tg**UJpXydc%P6DYiZMu zlCGmX{}|7Y`~SNCfV^lW-Zo_a=gj?a?cDyym3*6+7sC=*3id?SU&upe0C|9Plg1uD zu8ZXS0cfW`)DchG6Rr!{9Zeo=BkjrGK4g79<-@=?YNmWZ^C`Z~QNGdP@7AZ5bB~p< z1_su%9~QYDHozTVr@T92D4&eyUHd4XjOjhl4+Ai_zNV-B)6)Ji-`5>a`A(#KBm1y# zB91N4MLE3(*+bb$&7iKw+)MfBrhN9VUC#GK8yipC+QB_&n+K5n4)p(G71>VMhhP^x z3bqi+UodA5VxH4qjzXsw^V85qU80u#7Nn}bTu&RQo$?feKkcUf1wWquI7R;p=KjAv zfe!x}`WCc%NmklFI7HkUNQKE`oa=xbV{RZDIlhTK0_X?iscw8rMaVLk)Au87Qtw$# zACEO$+)QABfqlFKc!qg4_+dX#J$}tl%Rj_lMz%vY*uln{xYf$P$T2qsvHlIOVLk?J zJu9RYT6|gmhBt5s6W+q9x3@8}l)~ z>0$Qj(?3+-N%#G=y__4Mp7Gy&7M(cK%4x;U`|9x@(6fIZJh-``Hi$LfAMO_@k{cT; zwkXm+a)sm*zA2vc-$xiV9hj?b+e!LE3vTLJ1r?)9q!ffZTnYkjmHfCc$%E4AKq;9Dlw$3A_O6Aa3&0$B5p!r$H%f|roun486Ybb~ zNej6}(i3ixX6{?eb7bzhRkFga;=gC?hYP-1a;BC@Zpbxr?`6QhV9&Kun7>qt$mHVb zWl}P|TuKu;pW&>x6kfsl*B+jge5&L-P4TC^gI%{!n!SAJOHkh>F_z<;&_Tuwz4#f4 z^L6BLcmsqulB~BWZp=P>Pvym|r|r2xapzOtb*TAGeMg&Or%s*QkB!|-9NpBnJww!Y z&_}&HcOJ_G^`ViuU!?ZYKY3dTm;NpNDX)dng84l6=s;%Y|4#H4^lmQFpJ%D-DCm|D zE_k72`HL*0d-gMM5Pkq~*5tqDC4a_jk6S2jAy32mFbZdZZHJn*#Pcy`2j(%$3b`5k z_@kW5u9iQB3%f$DMmBH{_XgVfW7Gq*CD-6K$UYAH@rB~NkFlm7&yCxg^F;GE_!5?5{|0$I za^ZZ@gu_z!8g7LK+&3fJkPjexsONr-?AyOi`7N@8xd@I-({u%dDSD*myod?;4YYtG z_-5q~NLTa{<@yDpd4gxyg1nWq`~@#zI0BxQ7! zk{UZ-v`PGr5YlnWFnfuv#Y-|-kIv#{vYCI-TfOAJh$|PFH|8bz7v@U=Kbk0HzN3gR z#k=QG#=NAIad+7%FEP;Wmec-LQ2$oO&g1;lB}x@xs=*Y$L@_(&NzD%S;%LJa%PQs_ z!|Ua%|0n8L#e29Tl%XvV3XiX8Wa!=uZR!vnzhm%M_y|tme z^bN?%KA`E)yoDcKQ_ZL^d5W9UmpW>g@8LXW(9Cf5%n91oiE9;i0Bx-o?e-XFTtPeI z`VMqOd)fclN7(L5oI!JeeW6kG9gp%KJ!C@!=SB{pV}f5ZvgNq1)Zkw0>nkSiVU9$v zAjDVduv?JUAkMqt87jE`F617l8ugXxjXtcCD`Fh>mEYo~0}Ic*VYR9XPTPh#m%RG| zzJxsTTp|DGK?&b~5%xC9;&1TFf>Nk~Miu=2#dmhqm3-eY=ySU=)V@I(T!}o3Tnx{_ z;~SKrY}LF`8CtSY89c4ZJC(tgRw_d$Z&C&~U|xf}{}yHFxx1CYrFSbscc?N-8G7wD zW$?Z&m?M?JO}8mS_1H6zmv2`F4{@J+kZX|(kZ-I~hSuDu3^v0(o0P$_P-WPr3f5_~Y)zy_qnVx&PTs%FxT3l$&^muSuW1eB&7~v~nHS-mEm@wx0AF zwD7zp!f(1!dD92`rp_K?0{~t!#a2iPC{Y$jmo>o7s%`1K(4R9 z$%BynHOUJ%MZ`-qaoM|^Nk zT}=63!g*uo7D)OQ&Wu~{D>{p>WXdbD0RQYY^udu!U@_RKOY)DsDh1mQO5sBZvK+TT zbl6HR`iY&2EblcxS%X^v_QA{4;oZ11=eSl4x14fEzu*qld=4}Hk4>1jz<gnt74}QVrL#@@LT#VJp1odfBGc9N2>R5R`QR` zCpl9}4vFWhBhr64Q3k&Kt!Vr}O)s1IUpQTVWZ@^!bCl#qcR;%U=&J0Cg!@7ra?P9JJ_5JD_HY*J+n-$L7kXpj1 z@ztqecQJnN;r{E<8&K`6^>v}&(T#q`+t_>k(EkAS|KWBjOZzs`W)owFuq)+V{QJ=z z8GM|xB`47B_=Na9=n|Yz+j{C(3*}O?S{XV-nJc24`-CY&pHpU@CENtzj?yL!b3Ox< zc5pT%@7G40UC;+=8n}>uf$G?wDt|=|sO~gigY+5I@8A#E|D?K&i2ND(H`RQKbpTaP z5pM8y04xS-gZ{N0?7_sAbQU0&gr>H$(>jzd9e%VTQ1i``pRbe#`bt@Z`u@IBj;sh^Pel;AaI}3@$Z9{@ z#Rfky&(Ka@qMh`jU8HTa63&LKjzK`qeNXJ;H;>67b?o*+PC(4igIezS8^^=qhzG8lb|0s7HmV>;H z1+#TKeH_gHfUHA4GD@3!wHNKnb{|=eTnTG{A*g0O(7n`j3Trvv^3O%g48t_N=UCsl zIA1=-{24gUlKz#=h&Hlrp3tVZQvFYoCY8nJXj_nFn8k ziFdO=J&ZrgSP$6_sEDC!L^%kg-N5XI*Wehu0fTT7PQw}a0M5cW_yp+kYc9f9a2dV@ zh7FqJ%ERJ=G+laBF^@c|)SQ1*sl600mJ9KmeS1J`#RsI$a{&F|1JV$d$ofbk^QVc@ z9FoY`KN0=UMD)`V*cT&f(tU_Ieein6Q4f)m#ZQy+aO%-@TUlDiDA-CZ~{hP3@*SWm;t|=>32XpXrUOa&;!GuNjM@&Nk=3ZQjA9=RewaZ9Y-V$(jjB) zi0B{_vSyA*_QVm?LuBr8NAolm#6Y1MgXW z{;*U)<@903=7*&UbM?ewF@+o!bLe5If!e6UVnJHD&W5auIV|=24od@ZH6oix*Jfl3 z_EuyY{_UZM*?&R$BfGf18`(oT_eTF-`iA4hK9eAh^YP-Gju+QRg19kzaPL1AF9Rd- z6odnk6m>w7cN~zEZ3iTE>jBa3JRoV)FH3sj0meVfYhcfeKEQv_4@x%S^u~jd)4{nt zAqOQd|1l*$?0^&`9gxDI1MC++AjRhoP!0}A>8WU?%zQu$`<_tBrw&NP&L@=0QwPMj z_n=fIJ*HF-9}pAc53~LWr6%I%O0C}kv8+14`}6(jo>1xxPbl?{CzOWx1JW4tdl@0W zMj>!L-wf=4-LMDtLLwxA0gPaU9@rf&ntfme2ONdtFbszHg>o6W<`(KP*ac6*)9@@L ze|}I>o_$pgVb(y}%dbkh`&DViY=_t24LAdThe^2ZR>l@^6BN93NNz^n4*3rqlAj_U zgkQto;2+R5cu4Bc9g>C*4#_{U|9*pLUWHt!dF_xCA}gR42JhLS49N~<@Y@F%cRZ*x z;noKiUG+kk{+K!6o~nfWSlR_$dm?j;T`mU zA&riII%J%mFFMGaX8a4;nDvmu8hS3|(NWKb0>-a}P!#ROe=qnC2If*IOJMv9<(Mm= z5grew0aSVqvj1{>x&sGnwi6dEx%K{I|W(2BVY+8wNKLgx}D2C(m-$Z6_q&z|n7aomNQM-+@h5!Ka3}m04!~hZhBWAeUg(E+;63;fzJXa-y-76p!Xxk)JO!DM16A-k zyagY^*Dwo8B;PnJglk|KtcJVbUbr8gf@k1)I1I^<202gwB~T7k&+1{g**rI ze^9_YKp_-u6e)%k#Jw8U!EF#l-(??dQ7nY6;eDP~#GrPR(JxBkEQREA^CTt1M^aB; zC)%QAlD3rf3a-gGwnB8WSTgH5Z>~8|vSqF4yKj~p`#Q;Whw*<1=B8if96ZK-u zzrenFt~1b&>c0qpLdA$*Jg&F@_wcgE&s-Z~(t?_G$b&0ZEd)hqZ8{-TD*JjXe2`KxNT6aGnf$XouV zhTo3)E5cy@yK27!GxPoG-tNy-JNyI2yM5%J$bZ4VVe$nZ@g~jY!{p;WGP%n~uE2Z^ zECUb8IDXkjUPo@kz5-b4(2Q65$bwMO{13E(dfod-pBpi6@sTj3#_vHTZTau#(khvB z?BAuxUAP+pX}1E%<7U#7jKesy(!Us-5}d zd}RN2MXK$J+)sSQZv1!B2DX0}-hjP76}JZFdb1UIhA*5AzdFU(Ko&SzweH#wUz;3qzk z%v{iKxrhBC*RURJh+~h|UStC6#>B0f)%*3|!#(k@V&ZP@d$oS=@4>zt|0VdduG0Sy zaYOkiVWIl?JQ?@L&5i#w{?D>r@fH44__ID!4OJt^VZ!+kCY1T{RmAxU=BV%7lJ6rt z?&`ZuR`N`~gb$i0>i*F3c%Bt|GS{m9<4}cLbpY{tefNyxcMunL^~`FuOiW(H9>Fzv ze1m-VRopL?G^jW}pL9j1@DOgy<5y!>zhUDp)=^JUPpa3R=Gr05gi-Hj7@g01a*c5B z=eVwgI2>GuSq-DUyPEfg2CjLW>yHt}KzKDSzK7%@?0WP#F}p-acXTLOPpaCF-URmK zN11!~q5KkG7iG-Q!QB5#w9&k8-zLJDS<|`$f1ag@IV{ya`J{S3bOsg^j%ROK!gX4n zE0#3i8Pq&g-%+g#Y$l#DkMQ`7^F6C);Eyxl9A_Xo&Y*FwKGAa~`F{!4z*@o*XO)Mryz^ZCv3@>xy_I|1fg9;>U_MOEhskdVOTDDNznZ3bq`$hq zm*MuDe;M|w5LF!>y}8hJb7`m6W9qk~enYMJPrgPR#FxC3xHoYR(!fByTTVS^P|M`Y z=qeMgI+S!GUc-6Zx|su`4l5txna^-NWjpzdx6BYOiM&z&v2*=R>a!4z=Sjsac`Z6A zixpXid&U;7StWd5A_uWguBGq8eJ0nF)*0MCp8E%h6i_b~pCsMGv7ZvjA#DE}=r2CU zS)bH%$@o`oC7v+!0`N;t{VpH+7jR$f^Vag*yj#@;(%>ZV?*A@cwVbK{Wj62(ljP+R z!nN^@Ci|dU_|S4`m*t#;${r`oRWZcBm+Rf%g`eho+D~1m=3P9}vt_v?qo^`)^m<89 z>&Whq@A8X1_Tv{r=Guuw+~@L@^qb2+-dm@-sd>ovpyttewLblT=jVHxUP?S$LS)r4 z#vquzf@$M0he0@Ogb0X)NsA&;s$3HyTag{dMSKb0h3p(e&u< za1Aks4IaX{VSw=cg!SXv0N4q8ArOLKa>*;+ld|1=?95K@u}$^L_>&3VI8v4sZIu-t>RbEr6P%=(3Em zABH})mAbUjeZU| z!3973x9bNzJP&Db6pq6Q7=lwU3?ncKV{mb$A`{5-Z~>;^5>#HgQ;air$~5NuJVj=Z z4X_P%sFFN(#-5QK$em#CxKkXk8}lBBh8WmQ*uBVoum^KA#K1m?g*ey`@sI$CkOW%L zfgbXq7}Pq~fQ%)siKnQ|h2mvc|NPCzc zXh0r?7ARpXH~GfhN)P6-qJJo3W$!5CMgLUB7ynTiE9zCo@{G!OYK=0!AYBROejzYv z?@}f=Whq9kIZJ$#D@&EhvuBm$bBe@bU%*_P^Z1A?LQ>o{%YY=;+z9L8HrNDr!F})m z?1C}PE6PRsj^j(7SCTagWXyD>GPdAq?!QPGU%Enhj4*e2bH6)&s!VRUUfGQK9@XxD zm2y8alxGVAx@ek5ARO~Xpjpv81qUD*G9U+vpd6^qG|xf=VIv_9^OM{w3iDQ=del%g zXsCuX2cZm}fdY6Ds0K8rLBkq&QVi|#9{RtuN2v+uWI!6`bjXOFCpt6Z87t!%&wR;7 z>S=FtkY95B=WrGZ{FX=|`?iawSbv8SD21|9w1H3_xkM_Em8|OHKK4b}XHFFeD z5ycz@7%^AED&9fu&*jsf$vZVq3PUO5P>f#@l!h?J0lF!k@%%i=LS|0S6MYA|TG(?N z=1Go&cf(zCfp?OdCow~%kI zlYVc($M8P*;MR`34689OgeuG$=mnZI%`Bt=O`)a{Nw-6jjl2fA9Q^pEKVvTJP3ALF z=hL?NQm$YP+z$7`gK%*@|Byw-z^|YH#<#!dU4a~X=tXY}l7H@a*C89B1@7O>+{Z(V z_4Y7seU9t)Qzj2GzRYF}TFrCSljKhBb&PeIA@2Vk^Id=AyLO?s(#o?~MKet~)G?Q6 zMoz%ijnoV9FwZ;87;PS9Jd<(FGuT707a*%}yJ{146W3{R`-bp0M&ggX1*)*SdFK=O zMZ+e{>mlDXPrl&VU^NVJ#^X0c`n2;dW2f)-9(#P5_xK05dh`D;@3AFcdXL@nuJ`yo zpTZx!y9m<@ldpW{J^AF9-VV$Ys{OtXy(h~;y~nzY2De$%Tmd3LLOftlmH7zbv0Pp(Y&)*M|bNux_8nYFPLNIkzuw2;R7 zVfxfk+P^DBcVU@iPFyEht3o6@GDP&4b7B`u?(j0n(_Sa}J2~%s+m%wtnr#uX7<&n_ zG?9PsAr08ekrlDm@!#7k#RyeU4W^@4iWzF47A%e{#0s{CE2M6S^Z76F|G^pd|0A2Q zH;=IYA6iea{~y{N?Ei<(SoZ%z_g?n@LvJMOzhDn!|35hC2mWvR@n4Y^|AcQ~2880e z62^S^-m$+6i?H7T55jYB1WJJ7rTH4JB_w&(@oAe@~+iVNu^ya(xa1jt6>?#JGNy?rEry$u1}9ykpjf!9s+wctAVFDQV&t7g9M7}yK@;38pf!Tm1S z4mZHh;FoY1UV;*M9ZrA^-i1-#_rL@2C_Dwfg(Hv#RbYiq z_%jT{-@%J#S_Hvx1Ka^Wg`dG6s0%*!p>6PGJqWIzM?C<$fKN`73KcL;IEI>|=U2--#TZ^GM<`D@54!hD4LO;8DsLlD=0h5H2NB~T5Yz_oA!YT$FY4hF#n|AZMpRX~%8 zj7E9`Y73gJumkRdrErFr%iW zS%rB%e!oWgBM%_&K>h?8G4}$hEQ~uAbHDF-IgmLM%nRTge6?66-EbBLVG}Hdm%isA z!ld%dpTGvp_>Dh}`5v%9H_!M6sIm+W!ISV1EP+oJQ@3o9B=)K&&uoztWGZ%TP?V&F zL`ga_ga4=NBBB`oMoAVj8@nF&9Q<>Ud2vyaA0H(JiBVFBEW%!ldkOxf$TGqkBBJP9 zM@a>;61x%iD*UUFrVCMGz7!=jGf`5Dv|zX5Zo|J0Sx@+e2==_Yvi!SPJ$wX#h&&FMke-1JidtUtAlFwNJ1;|3|MYtCS-z_D6cS|Xh zMcyq2D92m@mG~K<3Uf7>rtcOr)L^a!3w~CxVXlMv$h)Nh8ZkFPGkz`5in$Hi6FA=h zIx%-aH-0_Pi@6W%k(`eJPRuTF_@0nu z*bUn@v3|HoDv(8vO{^boV*PLv>xY{pkL&Wez5rS16)CS#2c3Zr;UXNMep*d^$&|E) zVY22?%tqv&!2`#r+diOedX={5E%3vBE8GvigoBU|f1!Q52!4cJ4H2*%egQ8+5){EV zfEt`;Dcl5i!ZvsqehyDVEEGc(#PE!dEuu|;%_K+jLfV)sxDNAOP|5r$b3MYkpj5-S zKm8co4>InWq7AG;KEYc4amIWO#$n|fBINUM+EMrf{(}BRb!vpX!+6bjUzj`-K^WG_ z--Cr*yMpoG0oqZ8I9lO;!aalE{kZ>@vEX^))e~JaGUB;{ zXXz*IZp||3Ne!0XjE|)+``p}IZbuRS3zenPRe2ryx}S^3GReBm7cyYEAa*$`4h|D> zaxjjI!N}lxzIa+cM#?a?w7?iR}qh@@Rvp359MAs5MD$;o!)b#5_k4&S&v#&aJNJL7gw^uH{@=a} z=)VlJKXH`(iO3(%pJh&{pST9VO#0}_hgs6BX4&yucRlZFp)%@gEB1aX~*Yf0xit~pG8-b#R=S>{HgP#yjC-J|} z^*0hv5MiTv$9nE{1pj++dkS7u^M5z%|6Z&$#W4TBhxPxR%>T!;{-4PDKQaTm4);vs z&5{+(`u`5r{~?Do%!RxY%n?Gt2=o6?gt-_>rkNvzGRy`j$FBk^F&m-Ef&LSic8~|f z%*}0Mj*e%rAg$cfhOFZm>X8lD8xt6#ZYBSr1zMpE+F3W~fKKRwZs>tt=mYiN91c+b z&EWz!c%UB!POx`^GthP)eFjQEqnQzMV9D4oXSmKa%cm zk&XvR`(=b1!L0@vK%Rt>{u|*c!aM*E6Rr`~5$_`IxtaTXM!cPb-%7mM#CaRxJ|dt0 z3-@Y> z{{8qha_@K4_vHGH$Qz_RkhBXT4WWnmi*D$HUT_4FhTsAxsQ)%}gZgj7elRnxHsy1U zUPA!;L^(IX5r7VRfYf2PAsg_o$9?WU#K@Ke>g4^b-NZ6Ki_8yZd>+X7Je=`4vN($I zIh0~A0YeDmd#H$Gd=GQu`y_OyV!s>TClSA@7@NZVl8Il-{Zl)bAHbf5)Zw3jdlum{ zk@^JArPv?9+)V)cW&&g#Y4b|0BDx_u$@(e;?9Lct-^FKi9jE?vY!> z!~6Afj{$HdtPmI2hd9Ru*u5ByO^X5P1>_zU4;lJhByOBNY z>HXpV;`R771gwyQJZlyF@PF|d&QGre6PUpUbJA=OZc z-H5%wm;J++*hl3@Sz&C#>^G*J)%YrsgiITvUGJcs$F4n2xt~T~h6(WwWcwIz7W`yCyEh_5ngxfmNL|HM}fril>tKhgg0 zqy0~${YP4{+onRKj(Vov%=|a}|NehRJgqIXAy($U_XbM`bL5>Plz;lI-Iw@w>BsgW z`>@-OQ~po!4et$MpD*7m;XF~-O8;@b2ha;2!wT~8VYrQN;0bsO zt|8x7!@Y1nJOVGm$8Zhb!UOO;^ui*(iFJ?(H}PGphDTv6jJ)Fe2!Qvndw)iM9j;-F zb|d^J!r~_ zA(Vj$tZ*ED0?XhT$bkSDy@vdN^DqhH@Hw1?tFI-U;S%OAfgzwqgI-z^{0nmm>*}dJ zoR3PFv<~!zAtN7sV#tIn$ev>T9daNS^77IDFGl~LNrFP+Dblk3&cs0plwvM(u>Q`( zK{@eObg=%;J&fqZsrqo$Tw~fhU(EaFb2dBw0*+<>Up)KI_Rkj^)M2jQ!TG>DIfH*2 z=L2tL&sPk4zoI!G822{Z+c$E)Zv^N2MzQ}alCyt8Ip22`=LZ^tUA!rDh-gj zImd&xCuf2^tz)!HM)v<2(EVyb|F;4C-yYhf4xybADaBl-LkCTd9+y_6B8l@a2yY~O z74Frzo8m>x``PQ1$o|#@5lieY#TvIuvF&A_D|%1$(d_w*VGO)Sq-i(jjO^t9I6L@1 zP88)olJXz6OX&#TrEvU*WJJd1;S@{ElHO4)@as*i&O}uA|U=utDEGKAQgx^6q)IcA62;&g7g!Tgd6MhE&4bQ?0@Cp<_ zDO7_68lV-rzz%rj!w`G{3;1rXg*zY;w!;0u z@8LC1fu@D>PkWl|q5LCLv1^B^|3|6+kr@|g|1VMh&(QuOv$5-O&%r+znMe5i9_oLt zFGLn$FW!UxUkv(x$g()}|7bJH6Vd-eR$@2eUWI=(($vBjvIqS?4}D3b1-o^a_3vQj z^Jv%WFZRFW)BcCB|HYU6FAmy&ut&20 zMX&Z@+5ZyAewQT59`q;B|A$o4AP*{`3EqGq_=5cZHypxDao1!4)q|!QD9)NLKqW)- z4fycw#lt~Jg$ei+uI2jMVH4a7yUC9L;=2mgff;Pj4;P8Yk852D2@-pF?B9iik0*LNY^ zGmI5*|KYqsJ9fte`Eh|c1!StPzi0#eB@LM#;?IltOHzZsBnSIT%5KK$oKc~N9LR-? zFn`en`Aa5Yvyeqm{!&c*CEULhndjyI-F_@&AyjfdBkooBS0l@?8z%T3xqk&x6YC>M zki6fA`oTw1jXt8)`bZk}Upg`aMRXnJOw3vMWh3>~yitIK0q|AEQ`)_n!#q{YjJvk85qErj)-A=ZE5SpSLVd^_UmW^CAldoQvN zyPbF(QLO(UUD(~Yd+_f^4q(?@qW?34o<1^#_AB)Q?LXtcG)RYxjf^WH6S5#%OaCW` zab^hPOk`f1IxeOEgL|Rz`~DC166~cO`aehm_Htwe{*_20_9|qxmNqw$wl|6I4Qjyx zRojzL~L z{U6?;fOfVJioCw1{+p%#o2CAnrT&{0L+q@SC(JUIos~+=MyQIOm1;0qXT{tyOCMyG z_HmZ>aaOFDZMfAT>kYHaq0LI;`B`bYFe}ZMW~F6nmNWcjIlpsO+V!*2Q9LW1gz18A z-K_9`dFsDe>GRBrT{}ztH_Q2av*I$&iW})knw5SS0NV)v0`jHbd2|7~k&NXzPofq~ zjHk^#3#8SHekL^2&uoFlDEgtH^h=Qq5%e1T5N+<&drd^o{54EyMpk{=x~_aAJo zcXO==ob*dwVCQ-V_B8se>5v-EK0Rb|0N)|;Y6AH#i8qISZZ2f+*_QDbN2~IPoKz>*4Um(?qoV&4?b2p$Idqo0s3C9=EF7fYe z^6+1v@GT^P3%cQE@_Q*<50AlrK@6OL>-i4Sk*|UUnji|dUGNGVfi$QCH~axmQP6x1 zlkgDraw4R{71Z&;a20T*oF){x5e`8L6v8hc7OKGp&%lGU2amu%Fn>~~8 z;U(NolE#nY_6+ha;{7{rS99Hd-0sCK5;Acc!|!XZX~pkace4Jsiu!*W`Jd<|e<4hbH}40dT=OpWVes%Q3CQ`p7uBC89rD2+dj)bS z(u!PzY(s89I*<{_x9RKbAsxz~0&eD=9|IE%!YA+<`~w0=i^cFeI0_ro^ny~ThIYQA z4!);Or1~G2Ze$PkUSwYk^S}F;|A}Ls7peXW+>QJz*KOuGVnO5Ukfbqm(;)>?p`Lf^ zg7ov~wvV8HKiwgjmpagK@1XzNA^J;d`;U(M47&07=R*PhmBd%1D^QB3(ElJzDU{)7 z819tv(N3wL-%#nG{f8>d)k)}om>K^=&8kkRwNek$CR!O++O~GdF!9yXRyGtY#_Z;?a1NA1dm#}@f*?XvGJ=D8M7j`%99^&dp4q(^B zqoZGJQj!fOB?VHoCPnKpDQO)hB^`5y!=&i)O-iQGq-5crjoaM(A9Y3UHr_vy_oshR zu#r6iP!x{-%P{ZHdzX&#{-mb?$}iCWAw4Te6C>$XMH*FOHu=&2fg1WfwO;7IgwX%N zY{RXN{z*M?Hb5iyQ2%4l9ESc2w1%SpLfH1b^nai;j{Z+9{hxUHKZ*2z_S65tY~Mlu z2b?^sYb*Vq82Ue$`*BnCUo;(z44*iT=+Pb_R+8M4bAdW)JMT~cgp@eHoiKEQO+;!OZ>)))IS4Oe^eV+Ai*0-z2 zSpQ~D-WiI_*n>J1zR}l-&|ME{Cz_(>)*(xK+eSpU~gjrdjaFw8yj>{+G8(D z2kxE3(?z&$=Jlx;q&J+oh)~A&!OT63@%>-m`=8+ZpZ1b7d{?6|a4AixV;$@x%zuZs zDW45-J&(RCC%!VbiG?LS_&22gq!uRJSD5gE!ly2ioVz_ zIpOV+d%RM~JMWhKqi;yT);y(f%BU0#yQP@$C1Y+W-I=G9?aNaPiFr!7!%g{L$oLoi zf6qes2hUs0gr}(m{r^zT|Kbd=TFwBo1T3WghilSFm z+y?E@=>Ho}NoVmX`oHfkR(g*hQr#b)Tv^b|viwoSCJ<#tsEd!w0F)T@t z96c;4kcwHGcv8~hPfB{+Ny*rAQgksVB{P0lvRZ~EyJuMRo?*#B=59SHc}Gw3{wJjX z3ZV#!p#(~y3=BgjrTq9wsTe&em1Dzv|D1m@!Wu4^hX2a`hY_h68If9~g*{-_p%Jkm z>w-t5e(#7hK;wxK&c7I8|HBCTA4WK5b41z*-|jae9YG^}|0B}X@;>AL5#IlO*8e|{ zzR^*!PmGG=(x^CRMrr>?#m&$EJl0Y4)kfta-$?S*`;r3xBtOo5LccBW4C9_Nv;*&p z7Ba)mNEWoAQ)*rNzT{xeg>2A6!QL}c23nLw#w^&aG5mXgHSZ=b#({fT zOW#euheouHF-J4@mg9+q0#+QAhKZJR| zr|Ao`|2>-xWBO(vnc+ImrA(!M)qpgF4@e_4MGa8D4oJ(M0cqVm!2I%nn1UCGnaR)^ z!qsl<5({Chp#x$I=#o05{d|`=hW^ZXAzk8f_=%gowueJ?`?vbDj>`FLkpr|(9`rK@ zSfBYb`k4dF&kRWKg)YVo1B@>{j4wTsi9&K#s7JCEfWJwD|3uy+&r%^l7#_dfSa=l*k7~XPx6BLB|orQDTt_6 z3InQ@BENnqhLZgrDTQRakCf8xI*yTskE6@<5V}lk@Jc0(+M_t@lKamGswxRYUc}!S z$e3>p>3EKDe>1vHV7JrHBtJ{oG{(8Wa*Z1H72+e$OV~`-#`V|Xwi3eNHnorB9G5-pF2PNQ&;rW`0e`q0n3V+51 z0n9bQ^KbxAo6sDBBajUFPzvQ>gq4(^KOx_PZ{gpth;=5W+%!zdX_n%D8quoe&m3)SL{=E zal|{tX?ROq$nL#P>4|enZ-SHlgj3psowT=3zGtU&`8xRrtW#R{_flS*(nh%YQ+C!X zoUGUMvR>g7%l;C@y0=8JA?rL&_WM*Q<|(JtoN!9*aVPUZ<%%)f$+-Y;F&|Q*lo79? zr$i}_bV`M;m-AGda(NZsLadYdq!J~M=g5z6(%*72zvV<9>n+hw^h!=}xsq#kN(Sh* z_DUu)E527!z3hzVoRSv%7V}w7`a4cZ;yIEVoRVV2o<#pQz$M8+eUjqkBD_nqtNJ7j znI7em3_q9XFlPq4m~V4Q_I{V>6J3%M>yq4kF3B@HBtO(81t;E?LS)e@7xjOilpsql z^-0#($+sb3nP$veAY9w~I36lJ--jO2^Ssr4!i|;*##&4(SGK z={w~jKV7W<_K6ee8gjAz>k<#{{U==Pp=U#+!6Hea8A@`4h4rs``32vt)?<-0Bkf;M zhLT~_Dmt@;^>2%06ZNe2MT($!Z@rWtOA{?p zw!jP%p%pH7BTI$iFv{zHGU_h7HJu?h}ElJY{Sehd9yiwtbDP(XD`l9x_NMy8Bf>HpM;HpD6%bD>EOv{FA? zMHkq>{GU}G$L&ebTE&9224^a^Xe+uTI;B3+%KPh-M()?- zXprU|TBU{OYmGW3Z9O`rJuy}3*lLx|1S|SKR`maLO3x{s(u?flx$JxE#BsqY&v8%J zm{r{1Iocro$QN+yi?d1ZaGKH+)+pWDccjbF$XLQA9r{LTM-HsYRQe}u^ci#t$9`yB zdYinAbU18czhskFFx%p7=q%Xe5M~Y3?6gsisI#EW&nB%C??^W0X1#?zoJ|@-ZH#Gb zq#bo=q)kd9Y*HLxqpYVZg@#O}An+Z@SN->;Db+1DbR`>^S7ePMoNtKlrMx9mF?ceS zGGuz9P13lZb_e(LwMmM{Cds~OO44*AgY0)C_fn(eY_o~}M5dC>b7oCtDw*6*7q3$? z;%qcVb&|BIPLd<*n0Kp_)ZKMb#=X*Z)balW`u^K>%20+*$!vI6vbNPp_SQPlo14%x zs*~K}CiFkwm2EdL|6}0UiKoa=$M~;KN>0^D=@@ZM)v^Acp_Dh&Nkv?iQhC0P^`9)I zYNQSwz9umxq$}p|I`$L1E4A7z#S)&TSOe?C#&gw$*0HxcOKC7?DUHac#5(37te{mh_ zKkv!_Qj>2&cPLj$K4D`2OSY1#Gl@1LTS;@6B;Bu>@xMmV=}q)cPSZchR+F=vzMU7Z5nOOg#|BtLcW#U_DmPTZgZ!`V>)6!DhEUmE` zr46@sKa+G2Pv=&XbnQ1uH?k)>SLqEl(f`d>>XkHP`jDCSq*ip{X330bVgIw4`p+zSuNKKc=7yLV zPnjiuuUQHb%u*O{mZCtj6ysjvXJ&oYEM-ZxVnCLkAC?MarI%TZV=YpJtR8M*{69?p zvPEj5la$&qvsj|dVhu8jjd^;yb6C2L>y_>}jncE*%=xQk>DynMx}wbD-d`)88ME|Hw8+4enK^?R z>15p0fo#7(nmJmfH?Br{^sUm3bnUDW=cQ!D;n&Lg{d>%r)JVUlMm&xhabK#DTKsFG zTgAM!hPK@#b>yE7H)~w0SddN8HRvwXNCW2jZ8eP9YM8sIVJ^Lf|4h|LVR5SzV9#&J zQSyRnq|}h3lmx!V8EQ4?(kCkw!8KBjyCJ-Wx->Zv5yeC;PHIf-uBRbwMgZt!8YRZ0kVy0*P^&RW3|tI>u1pB`dk?wO5={XtU`7a`U z7lO9_Y=nOK2HU{Z5%zzBZHV$BU32(9oVJlWBfS6imW;JE*v7AM*e0%x$Rz2eJ{!R^ z*kGH9HrQr~=cw2GfkFBcA`*LZ4*z$A@BfG-&&~1t7m@Tw5y>2n;Qx+D?sNqIS40X? z_J1)#e?^4*y@)))_5LLHjXY-6thu7q0`iXC3%-|;a;2*$Ca)Ze3+v!Wj3;4>djN$HPn)N@p zh1(F89b?)1#Q%C9ebS_xVY>?Je@43hfpxc_Pu)QuKk=W~=DEFGZ?2sc`^dC74!+I( z+_cmWPK%5005`Gcg#qy%ep`G?e1Cm}e`nV#^k3rNq2o79CvL-W;roA3?1!!t$F=nT zp3B(k_fLuoxwjT=p8Kz)f1m#UH#0W>3C{nU^0wfkS4!w~$QC}r|G#pz;e9rI;u z1NNeA@My+1#E)P_+OFjMpOq2cl``5z|397U|FI z+~KS6CecTC=qmbZXJq2QRWg3)6Ee#_I>|m9dFd({Mv=i;`g$C;!IvDi0per(XQhAN zEYGa7+#5F7db%98?yYB}YuBuFJ|4DpB!{H^(pfx}4qGefTA~hH^9!>)1K4eiH_h_S z+^mET&GJshtOQ9HD9(!i(y+~Uf%E-=GvYbO_d(w*eL6!@PwedC`{IQ&VkfSDZQtN{d#%&O|H-K5_w1dv6Zbi7uMH3fiEpjr|8>;yzdeZS z>ulY7cyxqs7sslP`+!5}A@n3_K`)>_^eFn@(QAL8oO;T+FJn6~owwDU;P)Rn79H=3 zIEm|-cYTQejN{cq>}CETj`iPA#+!I1PoQ^EmS@##IaeAV=UMy+<3!#s%0A}rME9VB z=nxu`TltOkJu*UH*;=wiTkHK9 zTiZm@)_(ARmJYV-oS2fXqgxsOr(fpKR*wHI(s!g}>u3HL=A(g$tuly)vRfsBhJ9OP zT*&^wiinh$9@lYy`1k4`o9nH`n&Wl`%|^4fxeqRv`48}??JL+~ zH%;*TQ?@1SmrHU#_y3{GCGEOgGN{p0=hQY4)b2jJI?BjR&{iP2Iw)(A;Jlpzh zZnN&rX|X^08F3t7-2L&-Nd47$n~USceKp_zRgV8_^EThTKitzbRnrP6li}A@2WulhTL!c_uuD z2CkiuK{WKzq(qK#|9>c88+jps|8r8t(D)4#-2c(Gmrd~gH_w0fOv=oGfNl2ifNk!P zfNh>@RTRYzu>Vow*$GLa)J>C;z9GPOc)*suG+@iYoJWPD6H-K_hbN?r9DHdV;d|m) z6hbA~JY75SG}GtB{0j4d?B`^HY2Ss1Pk6*J>CXRkiW2fkBs5ATE<9@1o)FZphgHl_=_^!T4(tIJ*|_Lp{u!@h%S!w#vRxlUZCuM>B6 zhj^NHh_`JAzwvdQv~x`eM6Z*e{W=MuaNiDTV4FtLHC1;=b8?5YoZKO;7k2Rcw?o?B z-60((c1S1lUCej0uBU5d=>OQk{r?W`-FHZA-wui2ze5s7dH?f)9g=!v zholefkPP!#=5wseU%f*{&S0X-n~V< zm0QI3{w@6e(=Bp=^p|j*yA*vAeHJw~@xKL8GipJt(OaYqwI^@k`)?1wO|wV39@xY0 zKkSj7tM^E6*B*}lJ)GOO%CUpD^89zJ3?9Fg=RdAfyf8L=;#L{qhwMiyx6=P}tBfai z$wYRSOuoNMrlQ?4eX?CzwO-rw#$-f|ChGQkbS${Kz_qB*UL@B`_Wj_c6o$& z;)Cn?{|nbs{`E3#zg}k0Y~Oa7W1IQxc8ONEODwrv9wlw!!gj{{wlm(hUD7AE%RMa1 zGJl9T-?g2w2F^9~0y>K7_FpgdkG6~BlI`NWdb=E9nd{-}#l3$!<1pLB``PXIe6FYc zujl^tdiqhW=lSz`{Ab&xfo&Q|*K~NhH1FHa^=G@Z9^Ec&4{XPOwVm;|?b6A77xUe$ z>$!S6->o-G-=)9h|1aDu$57qOU1Cq(#WU1hJpc0h7tFh|EBN2}{fiU){>6v<{>4Z9 z{zcaceJA{0!6jd&{4Yy5v?2}rw{ZR6!vDWv9N=O6e-CWIhqMJB(pRLN>wL!xTcnfa zUDs~m{+H|jzQ31V;=a@Cy#LJg|C%i_P~9SfA8wH$%8C#Vf0pb2g};~4Ygc&wUE%rv z?`g9wGD%rehhDDv|6W+(_}?OPlret;*Z*Dkv+vv@andA6oBRm>+XVhM^yl%|qm0Kg z23ok}QYoSmDkJmH4ek5zUENvpU0FV?34B?MiFh7=lixqV|G@aW9|ahX4_-Jfp`-X8 z9>)K`^{(;oaqdMK|G$%K6tOk_zr^_eBmY1CUw(I_|6%-?XkY@rDH_V+|3kw}kD$@h z`2Wy&6~8H(ynz4bBm6&jd}i?a%o5KXS&(@=fKl}4|DVlt^2EBN@cN`j@c$iOXFMAJ z&td#OsBjej4=OQTMs>pdC&%1Ns12=G`FL(+YP_%G3S&_6Q&L*gHy+qsvWAnj93GvrZMkG3ejM~E*Y?j`O-185#4 z&f0WBj(@=x+QaotCpr<{xDnchJEjQ%H{gS(6TuFOX#q?5P{ zj{b-F9T3!u`cSi--=#TR7wAUnaXs<>g7%;< zqwk?zEPsgjUXGJ*6Mr2YhwI-FKY^Y>kD(JR^AO)g`Z)22=p;JL@;A{*)W@}d5M5yT zIW)^Yy7P%0xk_IL@$Z;_jyQ7}{wS2=c|DH`OczmUKmK?$e`1OI>(}s? z;;%;uAO3ojVqF@IR~er_jz9l2|DR%FNv4WBmK-DW4`xI zd_Kg@pX7Hj@a{AcH?d7SamR=JP6lxs^ON)~Ox=Uehj?NizMez)c`XbX`LLZXXSBWnW zKS=yz^fUAv`iNzI;tA4^63-G}fnH`hPMqPq%@7w6KXXxcDOzUwW%L@VyX1Fb|A6Zs za-#arvR`Q%H?fEI^G%@Q#>Atf9U~rQ{Rr_8^AX~K3;6!}-j#mhW2hJPp>EWJI+6L`3D$cQ#I4W1 zD{aKhY}-O!jl@ml6((+QabH4ge3^dg5V+)B@lq%A+r;kQ@LR*Eo@p0yG%eH5NhfU; z-{aDMw66arISy>})BTU=Gid7AAPU(NJcrYlIHc9vbkvLWJ8^q*PwM^s|^ zT9)&bRrhyH{}x@x{68`OUzomuX&c}9|H$+wi2nuGE}vHWJ@+t4@AF6KYOx~tGW zrvF#szhe2d#J?bZ5q+NNzd+4Q-%8v~{FjRRcGAa~PN4%#=ZLT5zF;3|zJ%@}&AsS% zEdK}knCnj9|9unx@4NK>64x{D`Vjy3NBDn;vWzn zV%`1dM@&D09$XPjLBKx1%$-b_C|BvE2$^J(kj;MkNJM$ zWBV8A|Kt824WS4cMk8nxjiK>#-2bCLzyHVd44OrA=>PNne~|alP#R?(#g~P0D31!L zh)N?1QXX88oM{8vG|D&l^YjiVP>eiVIXKlk~}zr=KmcsKKD;(LkhEc-F> z&(PE8MdV^xf%T(I$B>Wt5K1xq8n4r$|J#rMNzCsz-9vs2TwCk7$FL(umE#B1 zU*P)#xtaDLFYA2Bf7x#&z;O{wa{O?dgozuNA0UrGG=w5(7>%G&G=|0z?_0_wnnKfP z2F;>5G>@VvhIl7a5-5pMD2>eZAIqfVkojLT1yn>OWd7HTY?tk?*^v{~BNuWb5Aqtl zR%eAKjrW9u(2oMB#mHLQV&%|ct!-s#tF^S%TH0zY9b@S@ipzF0e}Ym@lgH_x(dj0_ zkdZYNW-4q=H89m+O*JyrXib^4O{eDxn`QfXN0$k@5w%`Ud(Y`W8Bf?nU25-$94achUFIedvDleZ&t3f8mGd0rVhx2>l2>jDCzBML$80 zp`W71(a+Em=;!D#`UQFtJ%x^-r_nR$m*`pa9C{x8HF^QPi2ers3LQniM(675tk7YF zPAha-q1y^QR_L?BfE5OblR@i8TO;*@UxRx<_u4PPaYZcQc{RtC>%-TxGq^~rFOxRQ~VNZ_ld7N~km z4b{Bi(vD6O28K=8XX>NJOu2gA)bsa_X%n{9o3L~E_X8Iv-tVy{`b}H?K4P}|{k*B+ z@3W?E?>kM-?*~o$yx&ItH^>*|FYt)oM90g`=5w>$BDcz|H_ELy%dPL(>-Wg58;-2( zl3h2-uA61ot$a*GOPH|NOK2q^g;~GLTCvMovCCSqTXyYc#cpfGZfnJEYsDVfwTBgZ ztQC8#MSEq}UKZ`O7VWhb?UP;mShUYtw9i^}o9w!cMYmatZnGBMF1v1L(e2iv+pR@+ z$gVqBbceO*4r|eV*|ncV`>jR$twkWpE&>BaL}e{#6H*yM$|s~qLP{YdV?y$>vax_n zR=Z{QjRNWJTltvCtln)k&hA~Z+e*FLYLeX~wVGqM)eyV)$Zo3IU z!0vsr+k)SHo9wob@i^?ZK)GP*WO!Y z?=IQ9TlVgey)4`(dvBAyx69r;Wbc02Yh_>&?KPWPRQqJ#jlzDV3w|H#Z?nQX!~)%C zQdxWYHo5IaVOLw&x0%J(-o9OKzfsuTw_BC6n&Nh|(%SEL$Q?HdJKk!TJIrEh-|v_G zHwru7YNq{Wv9(AUQUuW5qPsrVJ_q}rWkLB)P%iT*wRFZq-8}f|^te)^RB^a!8b zh^4=lpVE?#%i|BqJf|IV^|G2mL}0KP`t} zl*7N0U&xd4Dv z)H6@WQ@@g@X#68WDiR$zAV(gQBM-@upU9CX9+Y3oFHOV!@__vE8}du$ACg}_Cck`Mo|Wh1 zxkrC$WaV`+-Yw6|^Uulie=UD4FUX7X;$c3P4$I%jujE&U`5cy`@@wIrO3O>~@{qh7 zk(Wp0RdiWx*v2Zdvfif>#!Nvf!76fGh-MAtVc7 zS!j@jCRu2fg%(+8m4!B0XqSa9S?HF99$Dy>g+5srkcB~67?OpEEDX!Sh%AiB!niC< z$ik#7Ov%EuEX>HltSrpQ!n`a*Wg#XDaal;nLQ)n|vXGU9oGj#Jp&$!ISt!e5oh;gA z(IJaYS*({umn^zv(IbmqS@g-GUls$h7?j13EQV#VK^B{3u|*bJWwA{b+hwss7CU9J zOBTCju}2nrWwB2d`(^Q%EDp%xpezo_Vnh~)WpP9nM`dwL7RP0ALKY`waY`1aWpPFp zXJv6t7UyL#DvJqOOv++P7SpnrmBpMa=4G)Ui$z&1$&yo+>Sf6#OKw^6$dXr<0X)TsvNRw|gR(Rv zOA%QbmZcF{8keOBS(=okDOsA9r5Rb8m8CgZnwO=hEX8CgE=vhnO3G47aOf{(WGO34 zIl+m)R1m(QIIUg@%PWoY%7DBwD6dS(D|7P7yex-gIV{T!vfL=kO|sl9%Pq3pD$8xM z+%C%75d*pbp96u(<2jzG~j*rOkQNjFwd_s;-%JC^VJ|oBH)EYL{0X@~Tr_^~tM#c{L!f2IbX8d9_JiZI)Nta&?2jDS@p`QPgec18kE(rtTxDMqpUW`YO}18pjB4eWVKyZJ7l#}R=Z@iTUL8y zwO3aAWVK&bkICwQtPaX*L{^7obwpN2Wpzwe$Axda)k#^MlGSNhosrd9S)G&Bd0CCh zYD`w+vYL?9q^zc7H7%THtd4eFHs+9zsu`r_d4fysQ>wwIr)$dA&|v zx6A7edEF_m*URfJdEG6qd*pSmyzZ0N{qlN1UJuIaA$dJ4ulLI9eey<~ykVC&obrZ8 z-U!MYA$cP#Z*<8US$U%nG#+{Ic1kq4ms5* zr<&wcvz%&?Q|)r9Lr!(esUA7iE2sM8RKJ`$CZ~qvR76e<%c&7LH7cjZYUX2RTKJf_m5+(rew#G`Q|%_`G(p#I6DDA)+XOv) ztk~q!%g4<0$y@y<9x%b635MjYh>3@50cl6g)R+mz`Iva(V(^yTntsbcWKEl?Nj_$M z43@VC|n80fSp9%Z~@0bOkDGkPPS^8CnlT>f2@>-5I3H8uxG9mq}3V${#mbjGLmzO_B+DdyDK`p^dEdeD=Qi7FuwwBl`Z_?Cr z(v&x8tvv0rHX|~b5Sa~6yR0pUO!lTqlcq|OwWTbZ;$ya(GPzBe+@?%!Qzq$@Njhbc zPMOU~HEr@HFw0D2mY%NPZUU>xNN-^gSSf48GgnJ#wlY&>Vlfg}v!+C7O|B*~DQeS9 zK{0EJo;6!p&0sZx)dX|$_B?%EAzO?Gj6IzU~OH4Vy?Cj*tnJ6RG0%KVGz8GBd}&`m1Vv*XH|}Avz>|9GFzKwD#yocn=@$%K8{Fh=1dBc zHpkTII?i2!uoawcGEsvS)TSHFwCR8&vVvwgZRbPw1T!X>wF1gGZNC_peC?D#z?6fJ zN$EIUPwNtqlAyN8)XmIU3C$)BGv(y-_LvDM#%WSIO)*XmRRXpk_&74j&4y01A?xe; z9wdmEV8jIEQ*ZJixEPsDnX@)9nb%v#pcRpknKN0L5(%ut7XvGCt+M8e&pNKmjuEro zWlAz9feB1X6Oji|ZM9XP#Z6F4XO4Xm5WPKDn=pCRwlZh5nXV-f|?d@|!yO zO|}FdM`lY?C(d3IkfGno!Q{?{v?N<*<#bMX>GgC;PM76Oot$yXnR+?nmNOnXf{}}yyK8}T*B3n zYvVf}3gq(mj!)hR$U8ydD#;Q4PK%r!ZaX_%AZo8g9kr;p77f;-p;|OSvZ>nSTrG;S zBq3)<2hWa@$LLTknzN#@fSer*%GvSuv*WBB@2Ew+wP>mq#cEN;ilzt8PLpza$ckpX za(0G1X2@f;5%N)kEFmeNGQTGUjF+G|lyEsCBkvQd!;c%l+jDK%P~mzu34 zrQTXJSc^t$(Nryp{dt>MZPWOl7sSnib9JL~&e<#UcOyG;ASbFvF62fY zArwXps1Y@xX4Hz>P&YDp_o03?fCkYJil8wxfu_(jnnAN@4#iMP&bg?Ui}GER=juS6 zs0)#oi@aRqy|JiOAnY{x0%&k-v-lUGpf4D905?36zv`Zn)iW zyWw@ig!>94|Vp?KAsjt-96OX zLtA)`A-40d-8AhxP1O~LAxf%D*PVR)&3;p}Y+HFhiLc_F;y7n1MM%`5Ee* zfjh%~%ur^AGBdP&28Im#HPeHrLxwtJXqya78TLtrHp;M1GVGHK`y@mCGVGHK`y?}t z*e4mjd3x5{=Ec{vcv+!r(&%&RDKMQ{r{w(}i__Oe5 z;m^XKg+B{_7XB>!S@^T?XW`GnpM^gQe-{2M{8{*UIs9A}{w(}i__Oe5;m^XKg+B{_ z7XB>!S@^T?XW`GnpM^gQe-{2M{8{+3@Mq!YM}*F0;m^XKg+B*>4*necIrww%=itx5 zpMyUKe-8c}{5kk@@aN#q!JmUa2Y(L!9Q-->bMWWj&%vLAKL9Gp2gb8zP1QU4$d5$IXH80=HSf1nS(P2XAaIB zoH;miaOU95!^X%t5`#H~k&aXG zKMy~@{CzGDe;)ok{CW8E@aN&r!=Hyg4}Tv1Jp6h13vd?TEWlZSvjArS&H|hTI16wV z;4Hw&W#wD}&H|hTI16wV;4HvdfU^K+0nP%P1^5c^72qqtSAeeoUjeQHTm`raa24Px zz*T^&09OI70$c^S3UC$RD!^5Ms{mI4t^!;IxC(F;;3~jXfU5vk0j`4KV!syHuLbsN zf&E%wzZTf9MR<$w7U3Q_dCPFxS!&buUr(5_K<8_Y!q4v9C((s}c+)7)mgdU?{;*f}zB|DzUFh zu#{ja!BT>y1WO5)5-cTHO6;=|JSFy738oVJtOQqyW3mKW3APe!CD=-^m0&BuR)Vbr zTM4!jY$e!Au$5pd!B&E;1X~HV68o+MUx|HJg0Tc6Hz?;yaF*C_C0I+amSHTzScb6- zUm3nKd}Y|maFt;x!%~K&O#RE$zfAqh)W1yq%ha_@UCY$9Og+ogvrIk9)U!-I%ha<> zJAk1Ty;=^H>pXaunt$PxVe; zoMO28(oXatR+2{ezQD2@^+ zh0-XCawvb^4zRs|igMoJJ@23m9F&07#yD-xJIL8VTRF(tF?8P9blzDw@1()Z%+z^j z+r_o#o$>R|ZkE>8kY-no?;!8M@>1d&pRfKtSmt>Q8USpG>a*Sc`||~#EfaK|5oP3DQIeq#g27EO@=N&O*SZL*1WbGxeD9>*# zH@WsPl|V@&Q>aw|YsT6uR?!BKBgMgy;z(O9adFpWSYXQH&~fCAtadZIvNq*q%21kP zsf5Y`tI@}`G`rpDKu%PTT*!?)MplWY+7}6(?0ToqYF&fi^q+Tzi2^8yXd!DpeEwpS zI~!QiU^S)5yV1x3V*gGt!#P|#fd5-{FCNe)Ip>zM#i*9JmC4rg9OfK}ramSE=FR@9 zP0`9u&Qt3Y`1m+873P$PSfv_}HfxPZ+HQ27n$o0>v05%2EaNnGcA9)0Wos*wdnc=F z?Zav4?6O+Vv?V8`Imu{VB6CQYIy1%YVYiqm4hD9Tv&YCfN=(FlH@ViPOkH}7tTId; z`;4qs<7lxKSi9A*a7sA)%__sq;aQupa_c{DTHUlbQyju`wJEB?Ap@T&=a^A#+lvc0 znyh8!Jgn^-Q!@@YCtot0Pv*q5qCX!4|Ic&G$=8cBf`*MOFjLFgl*x#16er&(PQFq2 zelk@UL*r-ynW8C|YG2%+%$Vh7OWM1Y3~N&~7`w_WHr1J~RglQE4`*5}XVV-8nwBv+ zGR5iYDP4%<^WIwa`?p98EoZYxJk?N zY3Q@5&}W0=5zU~qRNSO@ldqff+@w|9ZKxe}Aj)vFy_@v($yDfjskqt3<3mBzi~7+3 zVx5<~yiVjnZ13$ul;@?s-n>+NZbUi07$P6;s4D&*6hU#6kV=4U11t|vmjHPOC_g~n zfh;1QAZ$VM2~y7>Wdz|5W)S%Y;SbqSJtAF*d_ql#b_kI!M14clCp3>JFGRkfB4WER zZ4gdLrGff4c#$8$*l-L{zXsZ{fp%z^Ls7)`jnt!&w2d$|vM(CpZKN)Z)U}EHnpoFF zn{%F3nuic&wNOTDqg2{R-`0ZQXe0l2>f8={J8j-hKJAp%P8sc_@1R{f$ghJuIw`XY zrY`dBqP==(gC5$jhcbKFZ@tX-Qs-V6`vX!r7Dl8SppFC7dw{kdB)>u0V~~0ckIvLBW{y1O*mjQU&5`FE%{|BNo2P2?H0eBf&a-TuWzhk|e3U~d%I=G@O_U~x(G>JJ zSNMlvl_Ys4DK80flJ&`gR8kvLD)b&z=ryX)Yg3^|rb16lg&vp+Jtq}kVZK%+rP@$)w!c%yKr|$|+))k(tD?CA0czUiB$&YV{3QxKfo?t6HxmI`r zt?<-Y;pwqb29qblN}1*@XJwtIhxNK%S+^%;-C;+qvd(SKIyV68T>RGQfnImjBNGQv z7!9Ctl#_Kg^E}S2^Nq0Xomnmv!Bfl6B7hbuZh}2J7A)G>a05b-n;SJs$&h$71OtNl@c#6WOX_jd?rm4gX>1ROY3|^mM zcg~~{%`i)Wv+TxMDltnXW~mtCHtTaC#5Qx}!&$dJ&-8pk*171fN5^D62577->v0+= zPTDxl7pH7G*VhwllK?V7`ULe)QuidgElE9-l$)fn=xkq4Qx4-k>uHv!sSBOr>vUqT z)0w+YN9=lzhUOx=PKV?=osjEvIIh!~xK791IvsKAoI2}tvaQp(v`#0=dU*1{c%~mk-U#hPzib=(X7JG|C2t`UZ#k zhIbrIBJ%PkQ3~ZzSvKf?*zl3gM_xYC(gnfG0J1?B!v?3q23-mpbR}#AnCBGO2$ByM zrj1ZeHp2CY?ZcEE4xkYflZ^)I&&6k>k>wov8%?BXqFx;S8%QfMr^7}kES-Ib zJh~i+GP~gIqV2kApKh3YD3gm4uREYI#BxrAjb1yV?p&ZY`rz#&9T%yMe$pHx{W0?A zfZrITeFtgFp&)9KjR@%?Oh>5O@Fb#b80j>}{{~l@jS*N!s29Jm!fO?XGDfKT2z42y z%+U}c&(Q`%T}CsAZO6!$tJ208WsTED6Vz#fWfP>EpuYTS^Ts6GPtvAS)N_h$r`UI# zHyg90nIruiZ8}H!^W?`BXCq4eqtr7-d&O85CyrCEIBgM!lk;GMo6d~{+a{=U0_Ft! zKS^6AX@ex`Qsk3j+cf(%O+6TJ+Tf;igF%`NZZbC*blITOcY}`I4LWEy=rr9ZQZG77 zH|P}Ipc8b1&dm)j2^%H$3!QnJbT)0)xlj-_qquBx^Sw#8&8B@0&7%@3%cjGJ0%!)s zWRn5Q&3e{(I#C3Tqe;Yi5BYcsvgsv%dhj;sx!d%Tj`L#EdrUUzYTKl%ZIhe6O}f`M zIrcaG5$#0UpCTa6Y>dCRV zImNmu(o9kJDe{_P+iCKdrcN`oKX*=>v$WBCS~jEP$vFFFv<;CqM*GI7Lky-E9C5ab zQ%A<&HyMN9jMLr;V!o?4lhiB8vB7uqW{P%9(T;pKZ>DMQG|L%)+T?a-lR=(M1~N7o zyx3$QVv_-gO>S2<=}h0G6Md5o@J$BgH|g}Q)8=vJx(-)Sl2U-Sl`3?Ugmq@>hmJ%#l^kK#l6}`+CG?%Q3l_6)nfyQ^7-DY9xF?g zW2iddLaZO4-UFl^$V-)*^(xo)YJ_dLNv}r8Co+eq+c3+y^R9BtR7Yu-(J?eF)iDR6 z{l-dCouJLQ{#U2SbBgU51g}nyAo7@|p4c_2Gu?>!DD4vMkSf>hYMi!Dl3$8;P3NS_ zV04v1(<-;NRR%4qIr3s~vYMlg3>H=y2&^&?Smk7F891wQeXeqA zTP;vm2E?iyCshWcsti6=8EC39Xj5g-rpf?Jm4TNk123v~OjO^TsOhw*nP%aIIAli- z z6hXsi1dSro{wVyM=PC+6=eml*AB8^(e-!>G{M?_ZDEv|Qqwq)JkHQ~?pYI06uaznG zyNbdeg`ZzQQ&IS%@JHc~!XJg7>zsG z{5*lFDEv|Qc@k6ncOu30Qbpm9!XJe{3P0abDhfaUJwb6TRZ;k(@JHc~!XJY_27e6x z82mB#WAMk|kHOFVr;5QJgFgm;41S*aR1E$Y{4w}r@N&ze+>Q@ z{4w}r@bjIiV(`b{kHOFPr;5QJgFgm;4E`AWG5BNf^WCaq@bmquV(`b{kHH^=G_;b#C-CE@3O zRdJ7?lJF zm4rVDKjte%uZ&8D{>{8f!JmSk`*W3oKLvjZ{uKNv_*3ww z;7`GyfkbmS|HcnTw)!ic9Z;wg-H z3L~D%z>g76VZ>7y@f1cpMW?@_&p~0tQyKU%;wkQZR0e*GcnTw)!ic9Z;wg-H3L~Ck zkU(L?Q}hffjCcwop2CQyFybi&4irW_MIVO3h^H{(DFzZ0Mm&WPPhrGU3@Rv|@e~6K zih%`%5l>;nQyB3SJ&g(@o?@UuVZ>8B2P%ws3L~Dvh^KfaR2cCTMm&WPPhrGU81WQF zJcSWYW#Pw&r!e9vjCcwop2CQyFybkUcnTw)!ic9Z;wg-Hih&G;5l>;nQyB3SgBl7W zp2CQyFybkUcnTw)!ic9Z;wg-H3L~Dvh^H{(sVw{$@f1cpg%M9-#8dR$D2#XtBc8&D zr|30R81WQcriyz?g%M9-#8VjY6h=HnH>$#jr|3#m81WQFJjH#dqW?!>#8V8iD2#Xt zBc5WgMKRc-Fyg5k{21{RMm&WPPcZ{xn;LpL&{jOplM`6%Y81z&Qeg<_E_rQv-UWGwVVbD_;^i&Rh zo{tm*U6b3zoK~K>irZDI!x|tQvWs1Hr#UPX7zFcu%t}y5+ z40?+Db43@lq6=E_?4~g2DGYiFgPy{mr|6be81xi>L_!I^{g@I2oXsIyp zDGYq70Dl2~o?8?~K1Cm+qK{Ew50lG5RS6DHX#JisyTU(NAIYQyBdeMn8qoPhs>^3{xnK zeu|+Ah0#x8^i%Z3sv`Ut{ZtWt`mGeh7z)FmD#DNHPZiAum<$yrLq)&4!f2?<@R#8)!(WEK3_s>WRfZq) zp<<%4N}Yb;yn!$cgHa3%QX8d65tKQ2+%| z2!&AtYD7(_8MUBR)P~wo2kJy!s2la5Uet&B(J?fD2GJ0TpkXwEM$s4=M-ylgO`&Nt zgJ#hjnnzI-LvfTqNt8lqltEdPLwQs{MN~rY+u^svZ-?IwKR;Td?eNo|tL^aH;kUz2 z|C+{Mr?K#9JN(%Av>kq|eA*5_c0O%~A4{LM!;h^`+u_IBr|s}#@6&epvG{2_{Mh`o z9e(Eg zrw?0WCe-v}YX|%e_#N=m$EY3f)6b|K@H^nAzfn8j$B3dG@H^mV97tn8)DHL^@H^ng zjHn&(JK%S~?||O{KORGknNU06cfjv}AD5uUC8%)(Y6tud_#N;&;Kx^|9q`k)t})AL z9B~>)oW}H~F}-OlR2oB-#^j7?w1KB~2f&#0~$+Zi97yJz8YYah}A${$F z-vz%5ei!^M_%RA;7yK^xc?nGO5}4*CFztfh1-}b^7yK^xUGQTI(7YU`c{xn;a+tf>X+dKr<#|;G%tr~UJlc|9Hx0WO!IP>=H)Q$ zf*(_ocERt0AKQU;!S90K1-}b^7yOu(v>SemOWF-TFO_LtD${QG-SG2rnRdhPhTjdp z8-5H<8ViGV!;g(Y<4V^Uo;0p>%?oH6wntr;iLj9qKSO0)-lUY^q)_&xA@;O9j;%{ZUtWjgJF-vd8JFU^Z} z8oP()_%V=aY$VzPzXyH~{2ur{@H4)sJ@9+r_rQieyP3id*NpsQ+wgZ z%%;8YGp4D%@O$C+!taHjHx#rNe%?~hm=`sE0FCiY;|b7s0<;%?FZ@_yv=@Fa{9gFI z@O$BB%u3@M(D(+l7kW3bcs z4YUt_UU|?y__5w-AN&~av=4qC{P+>H4}Kr~*mATFejofk_{U7d ze*pdf`~mm_@CV=zz#o7=0Dk~}>{mJfe*k{QOLYML0Q>>?1Mmmn55SKhQwQMZwHM86 zFPhh0G_Sqzqm&4K#_n|h{s8;|_<0>h^E!;?br{X-FggH#0R8~{0r&&(2jCCDkI7R9 z;19qbfFHxB#_*{F@Z*co0r;_p=>Yse_%R3TApAl28QawuLp8=wjeSf9;Sa(egg*#> z5dI+iLHL942jLIGAA~;$e-M5Qr8)>drcxb*KL|hOQXPap2tS?#9fUs!e-Qp4{6YBf zDd-^lLHL942jLIGAA}z-j}F2ggg*#>5dI+iLHL942jLIGkI7jF;m7ZwgYaW$)j{}! z@MCP%LHM!6=^*@=Ty+S3jIKHae+YgIuNuRv4#6LSKLkJKSIv004#6LSKLmdW{t*0F z>omq#jkiMMtuZK zA^1b^W7*RfZFLC#5d0zdL-2>-55bQCSBKya!H*GFhv3Kaq(ktB;19tcf*%i*#`>p2 z@MF}~A^0)t>M;Ca_%ZEjY=Ih|ln%omh9AF_4#SUUN{8XcH>Jbyhv5&yABH~+e;EES z{9*XR@Q2|K!ykq}41XB@F#KWo!|;dU55pgZ9~+_$!ykq}41XAYyjeO7e;EES{MZzA z82&K)VfgW`=rH_Y_`~pr;m6CR!|;dU55tcES%={d!ykq}41XB@F#PylbQu0H{0;Co zz~2BrhGmT%Qe%hI4e&R>kAYd^k-vECD{0;Coz~2Br2K)b; zqdUo|vklg5+xNX|B>G8=!C;d)AqVL+>&A>e+;L9fZNMWfX;os(Z~n9YKl}f)-?9Dt zXa9fpyRn)7?00uI4x;Bj`~S25Kl>d=k2B11AU*%t?-pnNv;RN)|FhrG^!#W4fA;@p zzdN1z&wjT$^Pl|=spmiY|Fi!;`~S25Kl|Nj%zyU(Xa9fp|7ZVy_Wx)9iTx+`pV)t5 z|B3x4_PdFj6Z;)ykK4F8u^$+i6Z;)$&x!pf_Mg~)V*iQ#C-$G%?_4&p#Qqce zPwYRj--+os?w%9-PwYRj|HOW`cXML@iTx+`pV)t5|B3x4_Mg~)V*iQ#C-$G%e`5cM z{U`RH*neXGiTx+`pV)t5za#Q-L_R0>pV)t5zq_V6vH!&WQ~OWtKehkV{!{x;?LW2O zE#REm?*?j4?LW2u)c#ZZPwhXo|J43d`<=|rsr{$+pW1(F|Ec|__Mh5+YX7PIZVTts z{!{x;?LW2u)c#ZZPwhXo|J43d`%mq6^geD8=hS{!V@~Zqwg1%qQ~OWtcc(a~_Mh5+ zYX7PIr}jJFom2Zy?LW2u)c#ZZPwjWsJE!)a*?(sLnf+(>pV{x`an9^Nv;WL~_m6XC z|C#*`b?40fGyBi%KePYL{xkc}>_4;r%>Fa`&+K_4;r%>Fa`&+I?5|IGd~`_Jq@v;WNg zGyB1uIkO+mnKS$0fjP7P%>Hxx&+R|A|J;5j&U0@6x&7z%pWA&+R|A|J?p_`_Jt^x8MEfoZEkH|GE9=_Mh8-ZvVOc=k}l5 z@5Xe_?LW8w-2QX>&+R|A|J;7}cyn(5x&7|)=G=a_d2??6x&3g&oZEkH|GE9=_MhAD zc5lw@Kezwf{%`hwvmdXWZ}xw)|C{~pTj!ho-|YWp|2O-;+5gS{Z}xw)|C{~a?Ehx} zH~YWY|IL2*Y1{|SH~Zbl&NutN+5gS{Z}xw)|C{}8XXl&!-|YWp|2O;H(#|*gzuEuI zes{I=&HiupgG%$w{%`hwv;Uj@Zf@tB{qAnZ-R(F{pKtbmv)>Kwe6#Zn-=k{}9|AqY*_Fve4VgH5w7xrJ+e_{WH{qFMS!u|{UFYLdt|HA$Y`!DRju;1O@ ze7FC*{on0}XXd;8-|ctOKi}>DZvS`t-S^FR`@h@&-Tv?Pf4Bd;{cf7)yZzto|8D*W3vj56{_q=mu z|CRk$_Fvh5W&f4^SN31oe`UWL_PMg(?eFj$b7lXP{a5y1*?(pKmHk)tU)k?2blkkp zmHqDChyR!>`>*W3vj58dEBmkPzq0?z{ww>h?7y=A%Kj_+-ImUk{a5y1*?(pKmHk)t zgNt)z|CRk$_Fvh5W&f4^SN31oe`UWr<@sU%5BuHU&ky^5*#E=+ANK#S|A+lx%KWh3 zE$jTS-#zR6u>XhsZkz`qn;-W7u>XhsKkWZu{}20r*#E=+ANK#S-#zsFu>XhsKkWZu z{}20r*#E=+ANK#S|A+nVrUyNf7lPu%n$oH zg85;;o7?$e{}20r*zf*!_=5Ri{}21G?Z3AF+Wu?%ukF9K-;M5E+wWd`uI<0J|Jwd* z`>*Z4w*T7xYx`lHxwik>{%iZM?Z3AF+Wu?%ukF9K|Jr`2XRhtPw*T7xYx}S5zqbF{ z{%iZM?Z3AF+Wu?%-HXq){nz$i+kb68A2HYVU)z6e|F!+s_5(w6ZNJ;%aa%lYi^py8 zT-$$b|F!+s_Jgg%SImw5H}>Dye`EiR{WtbQM{{HUjr}+F-`Ia+|Bd}O_TSilWB-l) zH}>Dye`EiR{WtdC*neaHjr}+F-`Ia+|Bd}O_TSilWB-l)H}<=G9(T`kWB-l)H}>-$ z;|_Xm?7y-9#{L`oZ|vtl=Ei<7YHsYmvH!+?He_z>cRM{d_5)LMWB-l)H}>Dye`EiR z{Wtc*RKt?YPy2t`|I_}T_W!j1r~RPSfL8O<{-5^$wEw65ZujS>{XgyhY5!0Af7<`k z{-5>(lmlnYPy2t`|I>a@Yku1Q(|&ek*p>NdKfEAI_Yg_PY|EK*w?f+^2Py2t`|I_}T_W!j1r~NX|FZv={lD!0Wxt#A`DOnv`+wR0%l=>XgJ$!~{$KXHNuOW#gJ<*0{$KXH zQJ-J-|FZv={lD!0W&bbxf7$PTeSX>h%l=>XyJeqW_W!c~m;Jx&|7HI#``x(DFZ(%( z`DOnv`+wQ*j(dLD|I7Yg_W!c~m;Jx&|7HI#`+wQbR?IK^VK|>V@$CO)|1bM*?FZE7 z*8W@jZ|w)>hQ*j$`)}>Pwg1+BzG!akzqS9?{#*NR?Z37E*8W@jZ|%Re|JHu!Zf@RSmwg1-sTl;VAzqS9?{#*NR z?Z37E*8W@jZ|%Re|JMFn`)}>PwVx}QTl;VAzqS9?{#*O+?7y@B&i*_5@9e*`|IYq9 z`|s?(v;WS1&S38Bzq20(o;&;R?7y@B&i*_5@9e*`|IYq9`|s?(v;WS1Mq%#kzq9|& z{yY1bh2dR>*P1)~@9e*`|IYq9`|s?(v;WTiJNxhKzq9|&{yY2c?7y@B&VKG;0Oh%} z|IYq9`|s?(v;WTiJNxhKzq9|&{yY2c?dK%s-u`?0@9k$M=HC8$`|s_)xBuRLj$-cZ z2cYNP{(JlH?Z3DG-u`?0@9n?0|K5Jka=44RxBuS$d;9P0zqkM1e!ge!?Z3DG-u`?0 z@9n?0|K9$4`|s`NHs;>`d;9P0zqkM1eo=+Fw;#qH7<=yRzqkM1{(JlH?Z3DG-u`?0 z*^jxm|K9$4`|s`Ni{`=p2m2rFf3W|-{s;RX?0>NT!TtyPAM9t5=D~ihZXWD^u>Zk+ zc61)>f3W|-{s;RX?0>NT!G1`59_)Xx|G|C+WghJ3n&!d&2mAS^d9eS%{s;RX?0>NT z!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?0>NT!TtyPAMAgy|H1wT`ycFo zw4ZsMNBbY`f3%-}ok#m0?SHiY(f&vKAMNJ|=h1#Xb{_42wExlmNBbY`f3*M6{zv;C z?SHiY(f&vKAMJm%|Iz+O`ycIpwExlmNBbY`f3*M6{zv;C?dNpn(f&vKAMJm%|Iz+O z`ycIpwExlmNBhD1d9?r0{zv;C?SHiY(f&vKAMJm%|Iz+O`ycK9ZU1lkxw83f|8M($ z+yC4C-}e8u|F`|zW%wPNe+ON_uf9>aI=db;L?f+~4U;F>s|JQ!5cK+J` z*Z#lu|FvJ)VgB0BE)H)yf9?Nk|6lw6+W*)7zxMyN|F8Xj?f+~4U;F>s|JQzHi1};( zU;F>s|JVM%_W!m2ul;}R|7-tW`}xRWdgq`0T<`p||DXN;?3Y&?c5?pN|Ihw^_W!e= zrJR5E|FfR~9tL>++5gY}fA;^g|DXN;?C0m^pZ)*r=ji62{r~L$Xa7I@|Jnb~{(tuW zv;Uv{jQ#wx|DXN;>}T)hpZ)*r|7ZU{`&qpCXFq#9|Lp%~|3CZx+5gY}fA;^g|DXN; z?Eh!~Kl}gL|IdC!=J{tovpoOof3p9{e!g#>?0>TV$^Iw%pX`6K|H*#7d7kWlvj55c zC;Ol5f3p9{e!g^`?0>TV$^Iw%pX`6K|H=L*`=9K8vj55cC;Ol5f3p9{{wMqS)OoW1 z$^Iw%pX`6K|H=L*`=9K8vj55cC;Ol5f3lxToG1IA?0>TV$^Iw%pX`6K|H=L*`=9K8 zvj53`j&Yvs=V9mBeqDokw*T4wXZxS+f42YG{%8B2?SHoa+5TtypY4CP|JnX$`=9OC zMVM#%pY4CP|JnX$`=9N9w*T3FHhiA#f42YG{%8B2?SHoa+5TtypY4CPU)f}y?SHoa z+5TtypY4CPUo>f+?SHoa+5TtypY4CP|JnX$`=9N9w*T4wXZxS+7u}v``=9N9wx3g< z7yDoAf3g3?{uldS?0>QU#r_xjU+jOe|Hb|n`(NyTvH!*X7yDoAf3g3?{uldS?0>QU z#r_xjU+jOe|Hb|n`(NyTvH!*X7yDoAf3g3?{ulezT;|387yDoAXHMtE{uldS?0>QU z#r_xjU+jOe|HXdReqQWQU#r{|OU+sUj|JD9i`(N#U zwg1)rSNmV>f3^SB{#W~7?SHlZ)&5ueU+sUj|JD9i`(N#Uwg1)rSNmV>f3^SB{#W~7 z?SHlZ)&5ueU+sUj|JD9i`(N#Uwg1)rSNmV>f3^SB{#W~7?SHlZ)&5ueU+sUj|JD9i z`(N#Uwg1)rSNmV>f3^SB{#W~7?SHlZ)&4j8-|T<0|IPk4``_$;v;WQhH~Zi0f3yG1 z{x|#I?0>WW&Hgw0-|T<0|IPk4``_$;v;WQhH~Zi0f3yG1{x|#I?0>WW&Hgw0-|T<0 z|IPk4``_$;v;WQhH~Zi0f3yG1{x|#I?0>WW&Hgw0-|T<0|IPk4``_$;v;WP0o_%x{ z=FR>$``_$;v!8LFH~Zi1XWi%B{&)MeOy=GGcl+P%R}-3d`$fO!-G1TkdADEud*1DT zxBuOK272D@f4BeL{&)M|?SHpl&}`oAf4BeL{&)M|?SHpl|6z1f=H32x``_(nspsAP zcl(u$=H32x``_(!bucl+P%XPoEVevW_M?H4K^HhaX% zM;&S2?fZsU5BqiZM|Xcd?EkR;!~PHZKkR3z=fi&e zm-(>&!~PHZ^#kU^eja>2?EkR;!~PHZKkWao|HFRGnE9~(!~PHZ6`SV6{tx>ZsU5Bopt|FBbzsQI-2)BaEUKke5)m{0pZ?fcmk@>R!%YId?`Lh4Z zel3#uvj5BeFZ;jj*Xx-t`@ih}vj5BeFZ;jj|FZwfel4H*vj5BeFZ)%t=F9#s`!#;% z%lzwH0A|I7X_`@ih}vj5BeFZ;jj|FZwf{xAE#?EkX=%lRlDZP{xAE#?EkV~ z#cRIo|FZwf{xADgz2?jQFZ;jj|FZwf{xAE#?C1Z7{~!K;`2XSmhyNe`fB665|A+q{ z{(t!Y;s1yKAO3&%|Kb0K{~!K;_WhyNe`fB665|A+q{{(t!Y;s1yKAO3&% z|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe` zfB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yK zAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSm zhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y z;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K; z`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{ z{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K z{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665 z|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&% z|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe` zfB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yK zAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSm zhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y z;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K; z`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{ z{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K z{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665 z|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&% z|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe` zfB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yK zAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSm zhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y z;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K; z`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{ z{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K z{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665 z|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&% z|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe` zfB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yK zAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSm zhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y z;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K; z`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{ z{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K z{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665 z|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&% z|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe` zfB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yK zAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSm zhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y z;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K; z`2XSmhyNe`fB665|A+q{{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{ z{(t!Y;s1yKAO3&%|Kb0K{~!K;`2XSmhyNe`fB665|A+q{{(t!Y;s1yKkN=PVkN=PV zkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PV zkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PV zkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PV zkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PV zkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PV zkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PVkN=PV zkN=PVkN=PVkN=PVkN=PV?|ZjR1N#r` zKd}G6{sa3D>_4#o!2SdK59~j%|G@qO`w#3tu>ZjR1N#r`Kd}G6{sa3D>_4!d|BwHV z|BwHV|BwHV|BwIg!2SdK59~j%pZ|~l@4)^8`w#3tu>ZjR1N#r`Kd}G6{sa3D>_4#o z!2SdK59~j%|G@qO`}zO)|M>s-|M>s-|M>s-{|@axwExilL;Lyv`2P;==l?sj|Iq$J z`w#6uwExilL;DZyKeYeQ{zLl@?LV~t(EdaF5A8p+|Iq$J`w#6uwExilL;DZyKeYeQ z{zLl@?LV~t(EdaF5A8p+|Iq$J`w#6uwExilL;DZyKeYeQ{zLl@?LV~t(EdaF5A8p) z|H%F$`;Y8Dvj52bBm0l+KeC_ykN=PV@5ufm`;Y8DvY-Et|BwHV|BwHV|BwHV|BwHV z|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV z|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV z|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|L@rTWBZToKeqqa{$u-( z?LW5v*#2Yt`TzL;j_p6T|JeRx`}zO)|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s- z|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-|M>s-{~qmsw4eWv|BwHV z|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV|BwHV z|L?c`zwPJ$Z^{jc`F+W%_*tNpL`zuNz5|EvA4_P^Ty zYX7VKulB#%|7!oM{jc`R03-vD3_vmf$p9n+kPJXF0LcI(1CR_rG62Z{BmzuW(A|GWL~_P^WzZvVUe@Akjj z|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I> zzuW(A|GWL~_P^WzZodpbG622X|8D=g{qOe603-vD3_vmf$p9n+kPJXF0LcI(1CR_r zG62Z{BmxnHGowEST%rE16VbHRRdTxfK>xnHGowEST%rE16VbHRRdTxfK>xn zHGowEST%rE16VbHRRdTxfK>xnHGowEST%rE16VbH{a^dl09FlP)c{rvVATLt4Pey( zRt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rv zVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d) z09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt z4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP z)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey( zRt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rv zVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d) z09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt z4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP z)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey( zRt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rv zVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d) z09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt z4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP z)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey( zRt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rv zVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d) z09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt z4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP z)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey( zRt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rv zVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d) z09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt z4Pey(Rt;d)09FlP)c{rvU=Qq916VbHRRdTxfK>xnHGowEST%rE16VbHRRdTxfK>xn zHGowEST%rE16VbHRRdTxfK>xnHGowEST%rE16VbHRRdTxfK>xnHGowE*hBl(09FlP z)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey( zRt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rv zVATLt4Pey(Rt;d)0QSiKBm0l+KeAs9VATLt4Pey(_Q?Jt`;Y8Dvj52bBm0l+KeGSG z{v-R3>_4*q$o?bykL*9P|H%F$`;Y8Dvj52bBm0l+KeGSG{v-R3>_4*q$o?bykL*9P z|H%F$`;Y8Dvj52bBm0l+KeGSG{v-R3>_4*q$o?bykL*9P|JeRx`;YBEw*T1vWBZTo zKeqqa{$u-(?LW5v*#2YtkL^FU|JeRx`;YBEw*T1vWBZToKeqqa{$u-(?LW5v*#2Yt zkL^FUUkzZ@09FlP)c{rvVATLt4Pey(Rt;d)0QT7aWBb(rRt;d)09FlPkL^FU|JeRx z`;YBEw*T1vWBZToKeqqa{$u-(?LW5v*#2Yt)c{rvVATLt4Pey(Rt;d)09FlP)c{rv zVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d) z09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt z4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt;d)09FlP z)c{rvVATLt4Pey(Rt;d)09FlP)c{rvVATLt4Pey(Rt@0)ui{>I8!4kGjPmmI05JiC zU;x67?Q#-YT$&LG@kXDtAAx0{p&8_)%CgEAB~^c&%ZdgN4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5DlQueSR81G=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeFD7d}4?AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fPJP=D17q>-wWzI!jFRNi}0yn9WVU;2!9AJ z?SjX?{rD?Cx5viU|J$(RF#kO2x5NFn^V5erKYzIM&d!@(J@s+Ozc=hWj`Z)&<4A9Q zd;gTbHr_ww{f?)cRJV4r^c5WIWFfheE7DGe?{_j8yp#F1oeYBP>=s~W=e?c%)9-8$ zac9#sJL|vQS%aW(e6%O!8sR%Z0&GvtJbN)=Dc<>IJC<<-||JT?k<{~cG0!6i`I`_<*Ij;gxpnDW>?FE zUH#wfiUZhH6aKDR!gtl%A$akd%KC1~l)^89qM-2Dl3w3UPGC0|{oPzCc5@2e&B*F* zX2Ev*!E4;Ks_&-XdN<7}J5*G5C=cwAr{5vTx3?j?3uJnmph2;SQ9*71(V zTRU`U@6b-WLqE|D4J5m(;O;KXv%Bl=?#9h`H;lWx7WCcqcI~d=XwO!9f-mgZIcU%3 zEB36@b}!0?dr{Nbi(> Downloading {} {:.1f}%".format( + tar_filepath, 100.0 * count * block_size / total_size + ) + ) + sys.stdout.flush() + + urllib.request.urlretrieve(url, tar_filepath, _progress) + statinfo = os.stat(tar_filepath) + logging.info( + "Successfully downloaded %s, size(bytes): %d" % (url, statinfo.st_size) + ) + with tarfile.open(tar_filepath, "r") as tar: + tar.extractall(directory) + finally: + gfile.Remove(tar_filepath) + + +def convert_audio_and_split_transcript(input_dir, + source_name, + target_name, + output_dir, + output_file): + """Convert FLAC to WAV and split the transcript. + Args: + input_dir: the directory which holds the input dataset. + source_name: the name of the specified dataset. e.g. test-clean + target_name: the directory name for the newly generated audio files. + e.g. test-clean-wav + output_dir: the directory to place the newly generated csv files. + output_file: the name of the newly generated csv file. e.g. test-clean.csv + """ + + logging.info("Processing audio and transcript for %s" % source_name) + source_dir = os.path.join(input_dir, source_name) + target_dir = os.path.join(input_dir, target_name) + + if not gfile.Exists(target_dir): + gfile.MakeDirs(target_dir) + + files = [] + tfm = Transformer() + # Convert all FLAC file into WAV format. At the same time, generate the csv + for root, _, filenames in gfile.Walk(source_dir): + for filename in fnmatch.filter(filenames, "*.trans.txt"): + trans_file = os.path.join(root, filename) + with codecs.open(trans_file, "r", "utf-8") as fin: + for line in fin: + seqid, transcript = line.split(" ", 1) + # We do a encode-decode transformation here because the output type + # of encode is a bytes object, we need convert it to string. + transcript = ( + unicodedata.normalize("NFKD", transcript) + .encode("ascii", "ignore") + .decode("ascii", "ignore") + .strip() + .lower() + ) + + # Convert FLAC to WAV. + flac_file = os.path.join(root, seqid + ".flac") + wav_file = os.path.join(target_dir, seqid + ".wav") + if not gfile.Exists(wav_file): + tfm.build(flac_file, wav_file) + # wav_filesize = os.path.getsize(wav_file) + wav_length = get_wave_file_length(wav_file) + + files.append((os.path.abspath(wav_file), wav_length, transcript)) + # Write to CSV file which contains three columns: + # "wav_filename", "wav_length_ms", "transcript". + csv_file_path = os.path.join(output_dir, output_file) + df = pandas.DataFrame( + data=files, columns=["wav_filename", "wav_length_ms", "transcript"] + ) + df.to_csv(csv_file_path, index=False, sep="\t") + logging.info("Successfully generated csv file {}".format(csv_file_path)) + + +def processor(dircetory, subset, force_process): + """ download and process """ + librispeech_urls = SUBSETS + if subset not in librispeech_urls: + raise ValueError(subset, "is not in Librispeech") + + subset_csv = os.path.join(dircetory, subset + ".csv") + if not force_process and os.path.exists(subset_csv): + return subset_csv + + dataset_dir = os.path.join(dircetory, subset) + logging.info("Downloading and process the librispeech in", dataset_dir) + logging.info("Preparing dataset %s", subset) + download_and_extract(dataset_dir, librispeech_urls[subset]) + convert_audio_and_split_transcript( + dataset_dir + "/LibriSpeech", + subset, + subset + "-wav", + dircetory, + subset + ".csv", + ) + logging.info("Finished downloading and processing") + return subset_csv + + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + DIR = sys.argv[1] + for SUBSET in SUBSETS: + processor(DIR, SUBSET, True) diff --git a/examples/asr/librispeech/transformer.json b/examples/asr/librispeech/transformer.json new file mode 100644 index 00000000..e8bc6935 --- /dev/null +++ b/examples/asr/librispeech/transformer.json @@ -0,0 +1,49 @@ +{ + "batch_size":16, + "num_epochs":50, + "sorta_epoch":2, + "ckpt":"examples/asr/librispeech/ckpts/debug", + "solver_gpu":[0], + "solver_config":{ + "clip_norm":100.0, + "log_interval":1 + }, + + "model":"speech_transformer", + "model_config":{ + "return_encoder_output":false, + "num_filters":512, + "d_model":512, + "num_heads":8, + "num_encoder_layers":12, + "num_decoder_layers":6, + "dff":1280, + "rate":0.1, + "label_smoothing_rate":0.0, + "schedual_sampling_rate":0.9 + }, + + "optimizer":"warmup_adam", + "optimizer_config":{ + "d_model":512, + "warmup_steps":8000, + "k":0.5 + }, + + "trainset_config":{ + "data_csv":"/tmp-data/dataset/opensource/librispeech/wav/train-clean-100.csv", + "audio_config":{ + "type":"Fbank", + "filterbank_channel_count":40, + "local_cmvn":false + }, + "vocab_file":"examples/asr/hkust/data/librispeech_unigram5000.model", + "subword":true, + "audio_min_length":10, + "audio_max_length":10000, + "force_process":false + }, + "cmvn_file":"examples/asr/librispeech/data/cmvn", + "dev_csv":"/tmp-data/dataset/opensource/librispeech/wav/dev-clean.csv", + "test_csv":"/tmp-data/dataset/opensource/librispeech/wav/test-clean.csv" +} diff --git a/examples/asr/switchboard_fisher/prepare_data.py b/examples/asr/switchboard_fisher/prepare_data.py new file mode 100644 index 00000000..0bfa7e45 --- /dev/null +++ b/examples/asr/switchboard_fisher/prepare_data.py @@ -0,0 +1,354 @@ +# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" switchboard_fisher dataset """ + +import os +import re +import sys +import codecs +from tempfile import TemporaryDirectory +import fnmatch +import pandas +from absl import logging + +import tensorflow as tf +from sox import Transformer +from athena import get_wave_file_length + +SUBSETS = ["train", "switchboard", "fisher", "hub500", "rt03s"] + + +def normalize_trans_swd(trans): + """ TODO: docstring + """ + + norm_trans = trans.lower() + # t.v.'s -> t.v's + norm_trans = re.sub("\.'", "'", norm_trans) + + # [ex[specially]-/especially] -> ex + new_line = [] + for word in norm_trans.split(): + if "/" in word and "[" in word: + new_line.append(word.split("/")[0].split("[")[1]) + else: + new_line.append(word) + norm_trans = " ".join(new_line) + + # remove , , [vocalized-noise], [vocalized-laughter] + # [laughter-yes] -> yes] (we'll process ] later) + # them_1 -> them + # remove [silence], [laughter], [noise]; ab[solute]- -> ab- + # remove ], {yuppiedom} -> yuppiedom, 20/20 -> 20 20 + remove_set = [ + "<.*?>", + "\[vocalized\-(noise|laughter)\]", + "\[laughter\-", + "_1", + "\[.*?\]", + "[\]\{\}/,?\._]", + ] + for pat in remove_set: + norm_trans = re.sub(pat, " ", norm_trans) + + # remove - that does not followed by characters or at the first posi; + # remove ' that at the fist posi. + # e.g.: ab- -> ab; -til -> til; 'cause -> cause + new_line = [] + for word in norm_trans.split(): + new_word = re.sub("^-|-$", "", word) + new_word = re.sub("^'", "", new_word) + new_line.append(new_word) + norm_trans = " ".join(new_line) + norm_trans = " ".join(norm_trans.split()) + return norm_trans + + +def normalize_trans_fisher(trans): + norm_trans = trans.lower() + + # t.v.'s -> t.v's + norm_trans = re.sub("\.'", "'", norm_trans) + norm_trans = re.sub("\[.*?\]|[,?\._]", " ", norm_trans) + + # remove - that does not followed by characters or at the first posi; + # remove ' that at the fist posi. + # e.g.: ab- -> ab; -til -> til.; 'cause -> cause + new_line = [] + for word in norm_trans.split(): + new_word = re.sub("^-|-$", "", word) + new_word = re.sub("^'", "", new_word) + new_line.append(new_word) + norm_trans = " ".join(new_line) + norm_trans = " ".join(norm_trans.split()) + return norm_trans + + +def normalize_trans_hub_rts(trans): + norm_trans = trans.lower() + + # t.v.'s -> t.v's + norm_trans = re.sub("\.'", "'", norm_trans) + # (QUI-) YEAH IT SHOULD BE QUICK AND PAINLESS AND (%HESITATION) (I-) ((WITH) + # -> QUI YEAH IT SHOULD BE QUICK AND PAINLESS AND I + norm_trans = re.sub("\(%.*?\)|\(\(.*?\)|<.*?>|\-\)|[,?\._\(\)]", " ", norm_trans) + + new_line = [] + for word in norm_trans.split(): + new_word = re.sub("^-|-$", "", word) + new_word = re.sub("^'", "", new_word) + new_line.append(new_word) + norm_trans = " ".join(new_line) + norm_trans = " ".join(norm_trans.split()) + return norm_trans + + +def split_line_and_norm_swd(line, filename=""): + sph_key = "" + speaker = "" + time_start = 0.0 + time_end = 0.0 + norm_trans = "" + if len(line.split()) < 4: + return sph_key, speaker, float(time_start), float(time_end), norm_trans + sph_trans_key, time_start, time_end, transcript = line.split(None, 3) + speaker = sph_trans_key.split("-")[0][-1] + sph_key = ( + sph_trans_key.split("-")[0][:2].upper() + "0" + sph_trans_key.split("-")[0][2:6] + ) + norm_trans = normalize_trans_swd(transcript) + return sph_key, speaker, float(time_start), float(time_end), norm_trans + + +def split_line_and_norm_fisher(line, filename=""): + sph_key = "" + speaker = "" + time_start = 0.0 + time_end = 0.0 + norm_trans = "" + if len(line.split(":")) < 2 or "#" in line or "((" in line: + return sph_key, speaker, float(time_start), float(time_end), norm_trans + info, transcript = line.split(":", 1) + time_start, time_end, speaker = info.split(" ", 2) + norm_trans = normalize_trans_fisher(transcript) + sph_key = filename.split(".")[0] + return sph_key, speaker, float(time_start), float(time_end), norm_trans + + +def split_line_and_norm_hub_rts(line, filename=""): + sph_key = "" + speaker = "" + time_start = 0.0 + time_end = 0.0 + norm_trans = "" + if len(line.split()) < 7 or ";;" in line or "IGNORE_TIME_SEGMENT_" in line: + return sph_key, speaker, float(time_start), float(time_end), norm_trans + sph_key, _, speaker_info, time_start, time_end, _, transcript = line.split(None, 6) + if len(speaker_info.split("_")) < 3: + return sph_key, speaker, float(time_start), float(time_end), norm_trans + speaker = speaker_info.split("_")[2] + norm_trans = normalize_trans_hub_rts(transcript) + return sph_key, speaker, float(time_start), float(time_end), norm_trans + + +def convert_audio_and_split_transcript(directory, subset): + """Convert SPH to WAV and split the transcript. + Args: + directory: the directory which holds the input dataset. + subset: the name of the specified dataset. supports train + (switchboard+fisher), switchboard, fisher, hub500 and rt03s. + """ + logging.info("Processing audio and transcript for %s" % subset) + gfile = tf.compat.v1.gfile + sph2pip = os.path.join(os.path.dirname(__file__), "../utils/sph2pipe") + + swd_audio_trans_dir = [os.path.join(directory, "LDC97S62")] + fisher_audio_dirs = [ + os.path.join(directory, "LDC2004S13"), + os.path.join(directory, "LDC2005S13"), + ] + fisher_trans_dirs = [ + os.path.join(directory, "LDC2004T19"), + os.path.join(directory, "LDC2005T19"), + ] + hub_audio_dir = [os.path.join(directory, "LDC2002S09")] + hub_trans_dir = [os.path.join(directory, "LDC2002T43")] + rts_audio_trans_dir = [os.path.join(directory, "LDC2007S10")] + + if subset == "train": + # Combination of switchboard corpus and fisher corpus. + audio_dir = swd_audio_trans_dir + fisher_audio_dirs + trans_dir = swd_audio_trans_dir + fisher_trans_dirs + elif subset == "switchboard": + audio_dir = swd_audio_trans_dir + trans_dir = swd_audio_trans_dir + elif subset == "fisher": + audio_dir = fisher_audio_dirs + trans_dir = fisher_trans_dirs + elif subset == "hub500": + audio_dir = hub_audio_dir + trans_dir = hub_trans_dir + elif subset == "rt03s": + audio_dir = rts_audio_trans_dir + trans_dir = rts_audio_trans_dir + else: + raise ValueError(subset, " is not in switchboard_fisher") + + subset_dir = os.path.join(directory, subset) + if not gfile.Exists(subset_dir): + gfile.MakeDirs(subset_dir) + output_wav_dir = os.path.join(directory, subset + "/wav") + if not gfile.Exists(output_wav_dir): + gfile.MakeDirs(output_wav_dir) + tmp_dir = os.path.join(directory, "tmp") + if not gfile.Exists(tmp_dir): + gfile.MakeDirs(tmp_dir) + + # Build SPH dict. + files = [] + sph_files_dict = {} + for sub_audio_dir in audio_dir: + for root, _, filenames in gfile.Walk(sub_audio_dir): + for filename in fnmatch.filter(filenames, "*.[Ss][Pp][Hh]"): + sph_key = os.path.splitext(filename)[0] + sph_file = os.path.join(root, filename) + sph_files_dict[sph_key] = sph_file + + with TemporaryDirectory(dir=tmp_dir) as output_tmp_wav_dir: + for sub_trans_dir in trans_dir: + if sub_trans_dir in swd_audio_trans_dir: + fnmatch_pat = "*-trans.text" + split_and_norm_func = split_line_and_norm_swd + elif sub_trans_dir in fisher_trans_dirs: + fnmatch_pat = "*.[Tt][Xx][Tt]" + split_and_norm_func = split_line_and_norm_fisher + elif sub_trans_dir in hub_trans_dir: + fnmatch_pat = "hub5e00.english.000405.stm" + split_and_norm_func = split_line_and_norm_hub_rts + else: + fnmatch_pat = "*.stm" + split_and_norm_func = split_line_and_norm_hub_rts + + for root, _, filenames in gfile.Walk(sub_trans_dir): + for filename in fnmatch.filter(filenames, fnmatch_pat): + trans_file = os.path.join(root, filename) + if 1 in [ + ele in root + for ele in [ + "doc", + "DOC", + "mandarin", + "arabic", + "concatenated", + "bnews", + ] + ]: + continue + with codecs.open(trans_file, "r", "utf-8") as fin: + for line in fin: + line = line.strip() + ( + sph_key, + speaker, + time_start, + time_end, + norm_trans, + ) = split_and_norm_func(line, filename) + + # Too short, skip the wave file + if time_end - time_start <= 0.1: + continue + if norm_trans == "": + continue + if speaker == "A": + channel = 1 + else: + channel = 2 + + # Convert SPH to split WAV. + if sph_key not in sph_files_dict: + print(sph_key + " not found, please check.") + continue + sph_file = sph_files_dict[sph_key] + wav_file = os.path.join( + output_tmp_wav_dir, sph_key + "." + speaker + ".wav" + ) + if not gfile.Exists(sph_file): + raise ValueError( + "the sph file {} is not exists".format(sph_file) + ) + + sub_wav_filename = "{0}-{1}-{2:06d}-{3:06d}".format( + sph_key, + speaker, + round(time_start * 100), + round(time_end * 100), + ) + sub_wav_file = os.path.join( + output_wav_dir, sub_wav_filename + ".wav" + ) + + if not gfile.Exists(sub_wav_file): + if not gfile.Exists(wav_file): + sph2pipe_cmd = ( + sph2pip + + " -f wav -c {} -p ".format(str(channel)) + + sph_file + + " " + + wav_file + ) + os.system(sph2pipe_cmd) + tfm = Transformer() + tfm.trim(time_start, time_end) + tfm.build(wav_file, sub_wav_file) + wav_filesize = os.path.getsize(sub_wav_file) + + wav_length = get_wave_file_length(sub_wav_file) + files.append( + (os.path.abspath(sub_wav_file), wav_length, norm_trans) + ) + + # Write to CSV file which contains three columns: + # "wav_filename", "wav_length_ms", "transcript". + out_csv_file = os.path.join(directory, subset + ".csv") + df = pandas.DataFrame( + data=files, columns=["wav_filename", "wav_length_ms", "transcript"] + ) + df.to_csv(out_csv_file, index=False, sep="\t") + logging.info("Successfully generated csv file {}".format(out_csv_file)) + + +def processor(dircetory, subset, force_process): + """ process """ + if subset not in SUBSETS: + raise ValueError(subset, "is not in switchboard_fisher") + + subset_csv = os.path.join(dircetory, subset + ".csv") + if not force_process and os.path.exists(subset_csv): + return subset_csv + logging.info( + "Processing the switchboard_fisher subset {} in {}".format(subset, dircetory) + ) + convert_audio_and_split_transcript(dircetory, subset) + logging.info("Finished processing switchboard_fisher subset {}".format(subset)) + return subset_csv + + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + DIR = sys.argv[1] + for SUBSET in SUBSETS: + processor(DIR, SUBSET, True) diff --git a/examples/translate/spa-eng-example/prepare_data.py b/examples/translate/spa-eng-example/prepare_data.py new file mode 100644 index 00000000..878e6892 --- /dev/null +++ b/examples/translate/spa-eng-example/prepare_data.py @@ -0,0 +1,46 @@ +# Only support eager mode +# pylint: disable=invalid-name + +""" An example using translation dataset """ +import os +import re +import io +import sys +import unicodedata +import pandas +import tensorflow as tf +from absl import logging +from athena import LanguageDatasetBuilder + + +def preprocess_sentence(w): + """ preprocess_sentence """ + w = ''.join(c for c in unicodedata.normalize('NFD', w.lower().strip()) + if unicodedata.category(c) != 'Mn') + w = re.sub(r"([?.!,?])", r" \1 ", w) + w = re.sub(r'[" "]+', " ", w) + w = re.sub(r"[^a-zA-Z?.!,?]+", " ", w) + w = w.rstrip().strip() + return w + +def create_dataset(csv_file="examples/translate/spa-eng-examples/data/train.csv"): + """ create and store in csv file """ + path_to_zip = tf.keras.utils.get_file('spa-eng.zip', + origin='http://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip', + extract=True) + path_to_file = os.path.dirname(path_to_zip)+"/spa-eng/spa.txt" + lines = io.open(path_to_file, encoding='UTF-8').read().strip().split('\n') + word_pairs = [[preprocess_sentence(w) for w in l.split('\t')] for l in lines[:None]] + df = pandas.DataFrame( + data=word_pairs, columns=["input_transcripts", "output_transcripts"] + ) + csv_dir = os.path.dirname(csv_file) + if not os.path.exists(csv_dir): + os.mkdir(csv_dir) + df.to_csv(csv_file, index=False, sep="\t") + logging.info("Successfully generated csv file {}".format(csv_file)) + +if __name__ == "__main__": + logging.set_verbosity(logging.INFO) + CSV_FILE = sys.argv[1] + create_dataset(CSV_FILE) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..d4ded8b7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +tensorflow==2.0.0 +sox +absl-py +yapf +pylint +flake8 +horovod +tqdm +sentencepiece +librosa +kenlm +jieba diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..134451f7 --- /dev/null +++ b/setup.py @@ -0,0 +1,67 @@ +# Copyright (C) ATHENA AUTHORS +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import sys +from glob import glob +import shutil +import setuptools +import tensorflow as tf + +TF_INCLUDE, TF_CFLAG = tf.sysconfig.get_compile_flags() +TF_INCLUDE = TF_INCLUDE.split("-I")[1] + +TF_LIB_INC, TF_SO_LIB = tf.sysconfig.get_link_flags() +TF_SO_LIB = TF_SO_LIB.replace( + "-l:libtensorflow_framework.1.dylib", "-ltensorflow_framework.1" +) +TF_LIB_INC = TF_LIB_INC.split("-L")[1] +TF_SO_LIB = TF_SO_LIB.split("-l")[1] + +complie_args = [TF_CFLAG, "-fPIC", "-shared", "-O2", "-std=c++11"] +if sys.platform == "darwin": # Mac os X before Mavericks (10.9) + complie_args.append("-stdlib=libc++") + +module = setuptools.Extension( + "athena.transform.feats.ops.x_ops", + sources=glob("athena/transform/feats/ops/kernels/*.cc"), + depends=glob("athena/transform/feats/ops/kernels/*.h"), + extra_compile_args=complie_args, + include_dirs=["athena/transform/feats/ops", TF_INCLUDE], + library_dirs=[TF_LIB_INC], + libraries=[TF_SO_LIB], + language="c++", +) + + +with open("README.md", "r") as fh: + long_description = fh.read() + setuptools.setup( + name="athena", + version="0.1.0", + author="ATHENA AUTHORS", + author_email="athena@gmail.com", + description="for speech recognition", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/didichuxing/athena", + packages=setuptools.find_packages(), + package_data={"": ["x_ops*.so", "*.vocab", "sph2pipe"]}, + exclude_package_data={"feats": ["*_test*"]}, + ext_modules=[module], + python_requires=">=3", + ) +path = glob("build/lib.*/athena/transform/feats/ops/x_ops.*.so") +shutil.copy(path[0], "athena/transform/feats/ops/x_ops.so") diff --git a/tools/env.sh b/tools/env.sh new file mode 100644 index 00000000..0cc092aa --- /dev/null +++ b/tools/env.sh @@ -0,0 +1,20 @@ +# check if we are executing or sourcing. +if [[ "$0" == "$BASH_SOURCE" ]] +then + echo "You must source this script rather than executing it." + exit -1 +fi + +# don't use readlink because macOS won't support it. +if [[ "$BASH_SOURCE" == "/"* ]] +then + export MAIN_ROOT=$(dirname $BASH_SOURCE) +else + export MAIN_ROOT=$PWD/$(dirname $BASH_SOURCE) +fi + +# pip bins +export PATH=$PATH:~/.local/bin + +# athena +export PYTHONPATH=${PYTHONPATH}:$MAIN_ROOT: diff --git a/tools/install.sh b/tools/install.sh new file mode 100644 index 00000000..9ebabfba --- /dev/null +++ b/tools/install.sh @@ -0,0 +1,4 @@ + +rm -rf build/ athena.egg-info/ dist/ +python setup.py bdist_wheel sdist +python -m pip install --ignore-installed dist/athena-0.1.0*.whl diff --git a/tools/install_kenlm.sh b/tools/install_kenlm.sh new file mode 100644 index 00000000..3295cafb --- /dev/null +++ b/tools/install_kenlm.sh @@ -0,0 +1,8 @@ +# reference to https://github.com/kpu/kenlm +cd tools +wget -O - https://kheafield.com/code/kenlm.tar.gz | tar xz + +mkdir -p kenlm/build +cd kenlm/build +cmake .. +make -j 4 diff --git a/tools/install_sph2pipe.sh b/tools/install_sph2pipe.sh new file mode 100644 index 00000000..15385d8b --- /dev/null +++ b/tools/install_sph2pipe.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Tool to convert sph audio format to wav format. +cd tools +if [ ! -e sph2pipe_v2.5.tar.gz ]; then + wget -T 10 -t 3 http://www.openslr.org/resources/3/sph2pipe_v2.5.tar.gz || exit 1 +fi + +tar -zxvf sph2pipe_v2.5.tar.gz || exit 1 +cd sph2pipe_v2.5/ +gcc -o sph2pipe *.c -lm || exit 1