Offline/Online (standalone) ESPnet2 Transducer

ci/test_integration_espnet2.sh

@@ -53,6 +53,30 @@ if python3 -c "import k2" &> /dev/null; then
        --asr-args "--model_conf extract_feats_in_collect_stats=false --max_epoch=1"
 fi
 
+if python3 -c "from warprnnt_pytorch import RNNTLoss" &> /dev/null; then
+    echo "==== [ESPnet2] ASR Transducer (standalone) ==="
+
+    for t in ${token_types}; do
+	asr_tag="transducer_${t}"
+
+	echo "==== [Conformer-RNN-T] feats_type=raw, token_types=${t}, model_conf.extract_feats_in_collect_stats=False, normalize=utt_mvn ==="
+	./run.sh --asr_task "asr_transducer" --ngpu 0 --stage 10 --stop-stage 13 --skip-upload false --feats-type "raw" --token-type ${t} \
+		 --feats_normalize "utterance_mvn" --lm-args "--max_epoch=1" --python "${python}" --inference_asr_model "valid.loss.best.pth" \
+		 --asr-tag "${asr_tag}_conformer" --asr-args "--model_conf extract_feats_in_collect_stats=false --max_epoch=1 \
+		 --encoder_conf body_conf='[{'block_type': 'conformer', 'hidden_size': 30, 'linear_size': 30, 'heads': 2, 'conv_mod_kernel_size': 3}]' \
+		 --decoder_conf='{'embed_size': 30, 'hidden_size': 30}' --joint_network_conf joint_space_size=30"
+
+	echo "==== [Streaming Conformer-RNN-T] feats_type=raw, token_types=${t}, model_conf.extract_feats_in_collect_stats=False, normalize=utt_mvn ==="
+	./run.sh --asr_task "asr_transducer" --ngpu 0 --stage 10 --stop-stage 13 --skip-upload false --feats-type "raw" --token-type ${t} \
+		 --feats_normalize "utterance_mvn" --lm-args "--max_epoch=1" --python "${python}" --inference_asr_model "valid.loss.best.pth" \
+		 --asr-tag "${asr_tag}_conformer_streaming" --asr-args "--model_conf extract_feats_in_collect_stats=false --max_epoch=1 \
+		 --encoder_conf main_conf='{'dynamic_chunk_training': True}' \
+		 --encoder_conf body_conf='[{'block_type': 'conformer', 'hidden_size': 30, 'linear_size': 30, 'heads': 2, 'conv_mod_kernel_size': 3}]' \
+		 --decoder_conf='{'embed_size': 30, 'hidden_size': 30}' --joint_network_conf joint_space_size=30 " \
+		 --inference-args "--streaming true --chunk_size 2 --left_context 2 --right_context 0"
+    done
+fi
+
 # Remove generated files in order to reduce the disk usage
 rm -rf exp dump data
 cd "${cwd}"

egs2/TEMPLATE/asr1/asr.sh

@@ -83,6 +83,7 @@ num_splits_lm=1   # Number of splitting for lm corpus.
 word_vocab_size=10000 # Size of word vocabulary.
 
 # ASR model related
+asr_task=asr   # ASR task mode. Either 'asr' or 'asr_transducer'.
 asr_tag=       # Suffix to the result dir for asr model training.
 asr_exp=       # Specify the directory path for ASR experiment.
                # If this option is specified, asr_tag is ignored.
@@ -203,6 +204,7 @@ Options:
     --num_splits_lm   # Number of splitting for lm corpus (default="${num_splits_lm}").
 
     # ASR model related
+    --asr_task         # ASR task mode. Either 'asr' or 'asr_transducer'. (default="${asr_task}").
     --asr_tag          # Suffix to the result dir for asr model training (default="${asr_tag}").
     --asr_exp          # Specify the directory path for ASR experiment.
                        # If this option is specified, asr_tag is ignored (default="${asr_exp}").
@@ -974,7 +976,7 @@ if ! "${skip_train}"; then
 
         # shellcheck disable=SC2046,SC2086
         ${train_cmd} JOB=1:"${_nj}" "${_logdir}"/stats.JOB.log \
-            ${python} -m espnet2.bin.asr_train \
+            ${python} -m espnet2.bin.${asr_task}_train \
                 --collect_stats true \
                 --use_preprocessor true \
                 --bpemodel "${bpemodel}" \
@@ -1101,7 +1103,7 @@ if ! "${skip_train}"; then
             --num_nodes "${num_nodes}" \
             --init_file_prefix "${asr_exp}"/.dist_init_ \
             --multiprocessing_distributed true -- \
-            ${python} -m espnet2.bin.asr_train \
+            ${python} -m espnet2.bin.${asr_task}_train \
                 --use_preprocessor true \
                 --bpemodel "${bpemodel}" \
                 --token_type "${token_type}" \
@@ -1191,24 +1193,24 @@ if ! "${skip_eval}"; then
         # 2. Generate run.sh
         log "Generate '${asr_exp}/${inference_tag}/run.sh'. You can resume the process from stage 12 using this script"
         mkdir -p "${asr_exp}/${inference_tag}"; echo "${run_args} --stage 12 \"\$@\"; exit \$?" > "${asr_exp}/${inference_tag}/run.sh"; chmod +x "${asr_exp}/${inference_tag}/run.sh"
-        if "${use_k2}"; then
-          # Now only _nj=1 is verified if using k2
-          asr_inference_tool="espnet2.bin.asr_inference_k2"
-
-          _opts+="--is_ctc_decoding ${k2_ctc_decoding} "
-          _opts+="--use_nbest_rescoring ${use_nbest_rescoring} "
-          _opts+="--num_paths ${num_paths} "
-          _opts+="--nll_batch_size ${nll_batch_size} "
-          _opts+="--k2_config ${k2_config} "
-        else
-          if "${use_streaming}"; then
-              asr_inference_tool="espnet2.bin.asr_inference_streaming"
-          elif "${use_maskctc}"; then
-              asr_inference_tool="espnet2.bin.asr_inference_maskctc"
-          else
-              asr_inference_tool="espnet2.bin.asr_inference"
-          fi
-        fi
+
+	inference_bin_tag=""
+	if [ ${asr_task} == "asr" ]; then
+            if "${use_k2}"; then
+		# Now only _nj=1 is verified if using k2
+		inference_bin_tag="_k2"
+
+		_opts+="--is_ctc_decoding ${k2_ctc_decoding} "
+		_opts+="--use_nbest_rescoring ${use_nbest_rescoring} "
+		_opts+="--num_paths ${num_paths} "
+		_opts+="--nll_batch_size ${nll_batch_size} "
+		_opts+="--k2_config ${k2_config} "
+	    elif "${use_streaming}"; then
+		inference_bin_tag="_streaming"
+	    elif "${use_maskctc}"; then
+		inference_bin_tag="_maskctc"
+            fi
+	fi
 
         for dset in ${test_sets}; do
             _data="${data_feats}/${dset}"
@@ -1250,7 +1252,7 @@ if ! "${skip_eval}"; then
             rm -f "${_logdir}/*.log"
             # shellcheck disable=SC2046,SC2086
             ${_cmd} --gpu "${_ngpu}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
-                ${python} -m ${asr_inference_tool} \
+                ${python} -m espnet2.bin.${asr_task}_inference${inference_bin_tag} \
                     --batch_size ${batch_size} \
                     --ngpu "${_ngpu}" \
                     --data_path_and_name_and_type "${_data}/${_scp},speech,${_type}" \
@@ -1261,7 +1263,7 @@ if ! "${skip_eval}"; then
                     ${_opts} ${inference_args} || { cat $(grep -l -i error "${_logdir}"/asr_inference.*.log) ; exit 1; }
 
             # 3. Calculate and report RTF based on decoding logs
-            if [ $asr_inference_tool == "espnet2.bin.asr_inference" ]; then
+            if [ ${asr_task} == "asr" ] && [ -z ${inference_bin_tag} ]; then
                 log "Calculating RTF & latency... log: '${_logdir}/calculate_rtf.log'"
                 rm -f "${_logdir}"/calculate_rtf.log
                 _fs=$(python3 -c "import humanfriendly as h;print(h.parse_size('${fs}'))")

espnet2/asr/espnet_model.py

@@ -14,7 +14,7 @@
 from espnet2.asr.preencoder.abs_preencoder import AbsPreEncoder
 from espnet2.asr.specaug.abs_specaug import AbsSpecAug
 from espnet2.asr.transducer.error_calculator import ErrorCalculatorTransducer
-from espnet2.asr.transducer.utils import get_transducer_task_io
+from espnet2.asr_transducer.utils import get_transducer_task_io
 from espnet2.layers.abs_normalize import AbsNormalize
 from espnet2.torch_utils.device_funcs import force_gatherable
 from espnet2.train.abs_espnet_model import AbsESPnetModel

espnet2/asr/transducer/beam_search_transducer.py

@@ -8,7 +8,7 @@
 import torch
 
 from espnet2.asr.decoder.abs_decoder import AbsDecoder
-from espnet2.asr.transducer.joint_network import JointNetwork
+from espnet2.asr_transducer.joint_network import JointNetwork
 from espnet2.lm.transformer_lm import TransformerLM
 from espnet.nets.pytorch_backend.transducer.utils import (
     is_prefix,