Merge branch 'master' into fix

.github/workflows/centos7.yml

@@ -19,7 +19,7 @@ jobs:
         # ImportError: /lib64/libstdc++.so.6: version `CXXABI_1.3.8' not found
         # (required by /__w/espnet/espnet/tools/venv/envs/espnet/lib/python3.6/site-packages/pyworld/pyworld.cpython-36m-x86_64-linux-gnu.so)
         # NOTE(kamo): The issue doens't exist for python3.7?
-        TH_VERSION: 1.12.1
+        TH_VERSION: 1.13.1
         CHAINER_VERSION: 6.0.0
         USE_CONDA: true
         CC: /opt/rh/devtoolset-7/root/usr/bin/gcc

.github/workflows/ci.yaml

@@ -16,19 +16,19 @@ jobs:
       matrix:
         os: [ubuntu-18.04]
         python-version: [3.7]
-        pytorch-version: [1.4.0, 1.5.1, 1.6.0, 1.7.1, 1.8.1, 1.9.1, 1.10.2, 1.11.0, 1.12.1]
+        pytorch-version: [1.4.0, 1.5.1, 1.6.0, 1.7.1, 1.8.1, 1.9.1, 1.10.2, 1.11.0, 1.12.1, 1.13.1]
         chainer-version: [6.0.0]
         # NOTE(kamo): Conda is tested by Circle-CI
         use-conda: [false]
         include:
           - os: ubuntu-20.04
             python-version: 3.8
-            pytorch-version: 1.12.1
+            pytorch-version: 1.13.1
             chainer-verssion: 6.0.0
             use-conda: false
           - os: ubuntu-20.04
             python-version: 3.9
-            pytorch-version: 1.12.1
+            pytorch-version: 1.13.1
             chainer-verssion: 6.0.0
             use-conda: false
     steps:

.github/workflows/debian9.yml

@@ -15,7 +15,7 @@ jobs:
       image: debian:9
       env:
         ESPNET_PYTHON_VERSION: 3.7
-        TH_VERSION: 1.12.1
+        TH_VERSION: 1.13.1
         CHAINER_VERSION: 6.0.0
         USE_CONDA: true
         CC: gcc-6

.github/workflows/doc.yml

@@ -30,7 +30,7 @@ jobs:
       - name: install espnet
         env:
           ESPNET_PYTHON_VERSION: 3.8
-          TH_VERSION: 1.12.1
+          TH_VERSION: 1.13.1
           CHAINER_VERSION: 6.0.0
           USE_CONDA: false
           CC: gcc-7

.github/workflows/macos.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.9]
-        pytorch-version: [1.12.1]
+        pytorch-version: [1.13.1]
         use-conda: [true, false]
     steps:
       - uses: actions/checkout@master

.github/workflows/test_import.yaml

@@ -16,7 +16,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: [3.9]
-        pytorch-version: [1.12.1]
+        pytorch-version: [1.13.1]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python
@@ -31,6 +31,7 @@ jobs:
       env:
         TH_VERSION: ${{ matrix.pytorch-version }}
       run: |
+        python3 -m pip install -U numba
         ./tools/installers/install_torch.sh false ${TH_VERSION} CPU
         ./tools/installers/install_chainer.sh CPU
         python3 setup.py bdist_wheel

.github/workflows/windows.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.9]
-        pytorch-version: [1.12.1]
+        pytorch-version: [1.13.1]
     defaults:
       run:
         shell: bash

.gitignore

@@ -82,6 +82,7 @@ tools/py3mmseg
 tools/anaconda
 tools/ice-g2p
 tools/fairseq
+tools/RawNet
 tools/._*
 tools/anaconda
 tools/ice-g2p*

.mergify.yml

@@ -13,9 +13,10 @@ pull_request_rules:
       - "check-success=linter_and_test (ubuntu-18.04, 3.7, 1.10.2, 6.0.0, false)"
       - "check-success=linter_and_test (ubuntu-18.04, 3.7, 1.11.0, 6.0.0, false)"
       - "check-success=linter_and_test (ubuntu-18.04, 3.7, 1.12.1, 6.0.0, false)"
-      - "check-success=linter_and_test (ubuntu-20.04, 3.8, 1.12.1, false, 6.0.0)"
-      - "check-success=linter_and_test (ubuntu-20.04, 3.9, 1.12.1, false, 6.0.0)"
-      - "check-success=test_import (ubuntu-latest, 3.9, 1.12.1)"
+      - "check-success=linter_and_test (ubuntu-18.04, 3.7, 1.13.1, 6.0.0, false)"
+      - "check-success=linter_and_test (ubuntu-20.04, 3.8, 1.13.1, false, 6.0.0)"
+      - "check-success=linter_and_test (ubuntu-20.04, 3.9, 1.13.1, false, 6.0.0)"
+      - "check-success=test_import (ubuntu-latest, 3.9, 1.13.1)"
     actions:
       merge:
         method: merge

ci/install.sh

@@ -20,7 +20,7 @@ ${CXX:-g++} -v
     fi
 
     . ./activate_python.sh
-    make TH_VERSION="${TH_VERSION}" WITH_OMP="${WITH_OMP-ON}" all warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pesq pyopenjtalk.done py3mmseg.done s3prl.done transformers.done phonemizer.done fairseq.done k2.done gtn.done longformer.done
+    make TH_VERSION="${TH_VERSION}" WITH_OMP="${WITH_OMP-ON}" all warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pesq pyopenjtalk.done py3mmseg.done s3prl.done transformers.done phonemizer.done fairseq.done k2.done gtn.done longformer.done whisper.done
     rm -rf kaldi
 )
 . tools/activate_python.sh

ci/test_import_all.py

@@ -26,6 +26,7 @@
                 not has_k2
                 and (
                     module_name == "espnet2.bin.asr_inference_k2"
+                    or module_name == "espnet2.bin.uasr_inference_k2"
                     or module_name == "espnet2.fst.lm_rescore"
                 )
             )

ci/test_integration_espnet2.sh

@@ -27,13 +27,13 @@ for t in ${feats_types}; do
     for t2 in ${token_types}; do
         echo "==== feats_type=${t}, token_types=${t2} ==="
         ./run.sh --ngpu 0 --stage 6 --stop-stage 13 --skip-upload false --feats-type "${t}" --token-type "${t2}" \
-            --asr-args "--max_epoch=1" --lm-args "--max_epoch=1" --python "${python}"
+            --asr-args "--max_epoch=1 --decoder rnn" --lm-args "--max_epoch=1" --python "${python}"
     done
 done
 echo "==== feats_type=raw, token_types=bpe, model_conf.extract_feats_in_collect_stats=False, normalize=utt_mvn ==="
 ./run.sh --ngpu 0 --stage 10 --stop-stage 13 --skip-upload false --feats-type "raw" --token-type "bpe" \
     --feats_normalize "utterance_mvn" --lm-args "--max_epoch=1" --python "${python}" \
-    --asr-args "--model_conf extract_feats_in_collect_stats=false --max_epoch=1"
+    --asr-args "--model_conf extract_feats_in_collect_stats=false --max_epoch=1 --decoder=rnn"
 
 echo "==== use_streaming, feats_type=raw, token_types=bpe, model_conf.extract_feats_in_collect_stats=False, normalize=utt_mvn ==="
 ./run.sh --use_streaming true --ngpu 0 --stage 6 --stop-stage 13 --skip-upload false --feats-type "raw" --token-type "bpe" \
@@ -45,12 +45,12 @@ if python3 -c "import k2" &> /dev/null; then
     echo "==== use_k2, num_paths > nll_batch_size, feats_type=raw, token_types=bpe, model_conf.extract_feats_in_collect_stats=False, normalize=utt_mvn ==="
     ./run.sh --num_paths 500 --nll_batch_size 20 --use_k2 true --ngpu 0 --stage 12 --stop-stage 13 --skip-upload false --feats-type "raw" --token-type "bpe" \
         --feats_normalize "utterance_mvn" --lm-args "--max_epoch=1" --python "${python}" \
-        --asr-args "--model_conf extract_feats_in_collect_stats=false --max_epoch=1"
+        --asr-args "--model_conf extract_feats_in_collect_stats=false --max_epoch=1 --decoder=rnn"
 
     echo "==== use_k2, num_paths == nll_batch_size, feats_type=raw, token_types=bpe, model_conf.extract_feats_in_collect_stats=False, normalize=utt_mvn ==="
     ./run.sh --num_paths 20 --nll_batch_size 20 --use_k2 true --ngpu 0 --stage 12 --stop-stage 13 --skip-upload false --feats-type "raw" --token-type "bpe" \
        --feats_normalize "utterance_mvn" --lm-args "--max_epoch=1" --python "${python}" \
-       --asr-args "--model_conf extract_feats_in_collect_stats=false --max_epoch=1"
+       --asr-args "--model_conf extract_feats_in_collect_stats=false --max_epoch=1 --decoder=rnn"
 fi
 
 if python3 -c "from warprnnt_pytorch import RNNTLoss" &> /dev/null; then
@@ -63,16 +63,16 @@ if python3 -c "from warprnnt_pytorch import RNNTLoss" &> /dev/null; then
         ./run.sh --asr_task "asr_transducer" --ngpu 0 --stage 10 --stop-stage 13 --skip-upload false --feats-type "raw" --token-type ${t} \
             --feats_normalize "utterance_mvn" --lm-args "--max_epoch=1" --python "${python}" --inference_asr_model "valid.loss.best.pth" \
             --asr-tag "${asr_tag}_conformer" --asr-args "--model_conf extract_feats_in_collect_stats=false --max_epoch=1 \
-                --encoder_conf body_conf='[{'block_type': 'conformer', 'hidden_size': 30, 'linear_size': 30, 'heads': 2, 'conv_mod_kernel_size': 3}]' \
-                --decoder_conf='{'embed_size': 30, 'hidden_size': 30}' --joint_network_conf joint_space_size=30"
+            --encoder_conf body_conf='[{'block_type': 'conformer', 'hidden_size': 30, 'linear_size': 30, 'heads': 2, 'conv_mod_kernel_size': 3}]' \
+            --decoder_conf='{'embed_size': 30, 'hidden_size': 30}' --joint_network_conf joint_space_size=30"
 
         echo "==== [Streaming Conformer-RNN-T] feats_type=raw, token_types=${t}, model_conf.extract_feats_in_collect_stats=False, normalize=utt_mvn ==="
         ./run.sh --asr_task "asr_transducer" --ngpu 0 --stage 10 --stop-stage 13 --skip-upload false --feats-type "raw" --token-type ${t} \
             --feats_normalize "utterance_mvn" --lm-args "--max_epoch=1" --python "${python}" --inference_asr_model "valid.loss.best.pth" \
             --asr-tag "${asr_tag}_conformer_streaming" --asr-args "--model_conf extract_feats_in_collect_stats=false --max_epoch=1 \
-                --encoder_conf main_conf='{'dynamic_chunk_training': True}' \
-                --encoder_conf body_conf='[{'block_type': 'conformer', 'hidden_size': 30, 'linear_size': 30, 'heads': 2, 'conv_mod_kernel_size': 3}]' \
-                --decoder_conf='{'embed_size': 30, 'hidden_size': 30}' --joint_network_conf joint_space_size=30 " \
+            --encoder_conf main_conf='{'dynamic_chunk_training': True}' \
+            --encoder_conf body_conf='[{'block_type': 'conformer', 'hidden_size': 30, 'linear_size': 30, 'heads': 2, 'conv_mod_kernel_size': 3}]' \
+            --decoder_conf='{'embed_size': 30, 'hidden_size': 30}' --joint_network_conf joint_space_size=30 " \
             --inference-args "--streaming true --chunk_size 2 --left_context 2 --right_context 0"
     done
 fi
@@ -90,6 +90,7 @@ done
     --asr-args "--model_conf extract_feats_in_collect_stats=false --max_epoch=1 \
         --ctc_conf reduce=False --encoder transformer_multispkr \
         --encoder_conf num_blocks=2 --encoder_conf num_blocks_sd=2 --encoder_conf num_inf=2 \
+        --decoder rnn \
         --model pit_espnet --model_conf num_inf=2 --model_conf num_ref=2 \
         --preprocessor multi --preprocessor_conf text_name='['text', 'text_spk2']'" \
     --inference-args "--multi_asr true"
@@ -133,11 +134,27 @@ if python -c 'import torch as t; from packaging.version import parse as L; asser
     cd "${cwd}"
 fi
 
+# [ESPnet2] test enh_tse recipe
+if python -c 'import torch as t; from packaging.version import parse as L; assert L(t.__version__) >= L("1.2.0")' &> /dev/null;  then
+    cd ./egs2/mini_an4/tse1
+    echo "==== [ESPnet2] ENH_TSE ==="
+    feats_types="raw"
+    for t in ${feats_types}; do
+        echo "==== feats_type=${t} ==="
+        ./run.sh --ngpu 0 --stage 1 --stop-stage 10 --skip-upload false --feats-type "${t}" --ref-num 1 --enh-args "--max_epoch=1" --python "${python}"
+        ./run.sh --ngpu 0 --stage 1 --stop-stage 10 --skip-upload false --feats-type "${t}" --ref-num 1 --enh-args "--max_epoch=1" --python "${python}" --local_data_opts "--random-enrollment true" --enh_config ./conf/train_random_enrollment.yaml
+    done
+    # Remove generated files in order to reduce the disk usage
+    rm -rf exp dump data
+    cd "${cwd}"
+fi
+
 # [ESPnet2] test ssl1 recipe
-if python3 -c "import fairseq" &> /dev/null; then
+if python3 -c 'import fairseq; import torch as t; from packaging.version import parse as L; assert L(t.__version__) >= L("1.12.0")' &> /dev/null; then
     cd ./egs2/mini_an4/ssl1
     echo "==== [ESPnet2] SSL1/HUBERT ==="
-    ./run.sh --ngpu 0 --stage 1 --stop-stage 7 --feats-type "raw" --token_type "word" --skip-upload false --pt-args "--max_epoch=1" --pretrain_start_iter 0 --pretrain_stop_iter 1 --python "${python}"
+    ./run.sh --ngpu 0 --stage 1 --stop-stage 7 --feats-type "raw" --token_type "word" --skip_upload_hf false \
+        --hubert-args "--max_epoch=1" --python "${python}"
     # Remove generated files in order to reduce the disk usage
     rm -rf exp dump data
     cd "${cwd}"
@@ -147,7 +164,7 @@ fi
 if python -c 'import torch as t; from packaging.version import parse as L; assert L(t.__version__) >= L("1.2.0")' &> /dev/null;  then
     cd ./egs2/mini_an4/enh_asr1
     echo "==== [ESPnet2] ENH_ASR ==="
-    ./run.sh --ngpu 0 --stage 0 --stop-stage 15 --skip-upload_hf false --feats-type "raw" --spk-num 1 --enh_asr_args "--max_epoch=1 --enh_separator_conf num_spk=1" --python "${python}"
+    ./run.sh --ngpu 0 --stage 0 --stop-stage 15 --skip-upload_hf false --feats-type "raw" --spk-num 1 --enh_asr_args "--max_epoch=1 --enh_separator_conf num_spk=1 --asr_decoder rnn" --python "${python}"
     # Remove generated files in order to reduce the disk usage
     rm -rf exp dump data
     cd "${cwd}"
@@ -208,9 +225,11 @@ if python3 -c 'import torch as t; from packaging.version import parse as L; asse
     for f in egs2/*/enh1/conf/train*.yaml; do
         ${python} -m espnet2.bin.enh_train --config "${f}" --iterator_type none --dry_run true --output_dir out
     done
-    for f in egs2/*/ssl1/conf/train*.yaml; do
-        ${python} -m espnet2.bin.hubert_train --config "${f}" --iterator_type none --normalize none --dry_run true --output_dir out --token_list dummy_token_list
-    done
+    if python3 -c 'import torch as t; from packaging.version import parse as L; assert L(t.__version__) >= L("1.12.0")' &> /dev/null; then
+        for f in egs2/*/ssl1/conf/train*.yaml; do
+            ${python} -m espnet2.bin.hubert_train --config "${f}" --iterator_type none --normalize none --dry_run true --output_dir out --token_list dummy_token_list --num_classes 10
+        done
+    fi
     for f in egs2/*/enh_asr1/conf/train_enh_asr*.yaml; do
         ${python} -m espnet2.bin.enh_s2t_train --config "${f}" --iterator_type none --dry_run true --output_dir out --token_list dummy_token_list
     done