Merge pull request #5327 from Emrys365/tse

Update several SE recipes and bash scripts

egs2/TEMPLATE/enh1/enh.sh

@@ -872,6 +872,7 @@ if ! "${skip_eval}"; then
                         ${_inf_scp} \
                         --ref_channel ${ref_channel} \
                         --flexible_numspk ${flexible_numspk} \
+                        --is_tse ${is_tse_task} \
                         ${scoring_opts}
 
                 for spk in $(seq "${ref_num}"); do

egs2/chime4/enh1/local/data.sh

@@ -139,11 +139,11 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
     #  (2) tr05_simu_noisy, dt05_simu_noisy, et05_simu_noisy
     local/simu_noisy_chime4_data_prep.sh ${CHIME4}
 
-    # prepare data for 1ch track: (TODO: et05_simu_isolated_1ch_track)
+    # prepare data for 1ch track:
     #  (1) {tr05,dt05,et05}_simu_isolated_1ch_track
     local/simu_ext_chime4_data_prep.sh --track 1 --annotations ${CHIME4}/data/annotations \
         --extra-annotations ${extra_annotations} isolated_1ch_track ${odir}/audio/16kHz
-    #  (2) {tr05,dt05,et05}_real_isolated_1ch_track (TODO: tr05_real_isolated_1ch_track)
+    #  (2) {tr05,dt05,et05}_real_isolated_1ch_track
     local/real_ext_chime4_data_prep.sh --track 1 --isolated_6ch_dir ${CHIME4}/data/audio/16kHz/isolated_6ch_track \
         isolated_1ch_track ${CHIME4}/data/audio/16kHz/isolated_1ch_track
 
@@ -152,6 +152,14 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
     local/simu_ext_chime4_data_prep.sh --track 6 isolated_6ch_track ${odir}/audio/16kHz
     #  (2) {tr05,dt05,et05}_real_isolated_6ch_track
     local/real_ext_chime4_data_prep.sh --track 6 isolated_6ch_track ${CHIME4}/data/audio/16kHz/isolated_6ch_track
+
+    # prepare data for 2ch track:
+    #  (1) {dt05,et05}_simu_isolated_2ch_track
+    local/simu_ext_chime4_data_prep.sh --track 2 --annotations ${CHIME4}/data/annotations \
+        --extra-annotations ${extra_annotations} isolated_2ch_track ${odir}/audio/16kHz
+    #  (2) {dt05,et05}_real_isolated_2ch_track
+    local/real_ext_chime4_data_prep.sh --track 2 --annotations ${CHIME4}/data/annotations \
+        --extra-annotations ${extra_annotations} isolated_2ch_track ${CHIME4}/data/audio/16kHz/isolated_2ch_track
 fi
 
 log "Successfully finished. [elapsed=${SECONDS}s]"

egs2/chime4/enh1/local/real_ext_chime4_data_prep.sh

@@ -6,6 +6,8 @@ set -e
 eval_flag=true  # make it true when the evaluation data are released
 track=6
 isolated_6ch_dir=  # used for preparing tr05_real_isolated_1ch_track
+annotations=  # used for preparing {dt05,et05}_real_isolated_2ch_track
+extra_annotations=  # used for preparing {dt05,et05}_real_isolated_2ch_track
 . utils/parse_options.sh || exit 1;
 
 if [ $# -ne 2 ]; then
@@ -14,8 +16,8 @@ if [ $# -ne 2 ]; then
   exit 1;
 fi
 
-if [[ ! " 1 6 " =~ " $track " ]]; then
-  echo "Error: \$track must be either 1 or 6: ${track}"
+if [[ ! " 1 2 6 " =~ " $track " ]]; then
+  echo "Error: \$track must be one of (1, 2, 6): ${track}"
   exit 1;
 fi
 
@@ -30,10 +32,16 @@ local=`pwd`/local
 utils=`pwd`/utils
 odir=`pwd`/data
 
+if [[ "$track" == "2" ]]; then
+  # no training data for 2-ch track; use training data in 6-ch track instead
+  list_set=""
+else
+  list_set="tr05_real_$enhan"
+fi
 if $eval_flag; then
-list_set="tr05_real_$enhan dt05_real_$enhan et05_real_$enhan"
+  list_set+=" dt05_real_$enhan et05_real_$enhan"
 else
-list_set="tr05_real_$enhan dt05_real_$enhan"
+  list_set+=" dt05_real_$enhan"
 fi
 
 cd $dir
@@ -57,6 +65,31 @@ if [[ "$track" == "1" ]]; then
     sed -E "s#isolated_1ch_track/(.*)\.wav#isolated_6ch_track/\1.CH0.wav#g" ${x}_wav.scp > ${x}_spk1_wav.scp
   done
 
+elif [[ "$track" == "2" ]]; then
+  # 2-ch track
+  if [ ! -f "${annotations}/dt05_real_2ch_track.list" ]; then
+    echo "Error: No such file: ${annotations}/dt05_real_2ch_track.list"
+    exit 1
+  fi
+  awk -v dir="${audio_dir}" '{ch1=dir "/" $1; ch2=dir "/" $2; print("sox -M " ch1 " " ch2 " -c 2 -t wav - |")}' ${annotations}/dt05_real_2ch_track.list > dt05_real_$enhan.flist
+  awk -v dir="${audio_dir}" '{print(dir "/" $1)}' ${annotations}/dt05_real_2ch_track.list | sed -E 's#isolated_2ch_track/(.*)\.CH[0-9]\.wav#isolated_6ch_track/\1.CH0.wav#g' > dt05_real_${enhan}_spk1.flist
+  awk -v dir="${audio_dir}" '{print(dir "/" $1)}' ${annotations}/dt05_real_2ch_track.list | awk -F'[/]' '{print $NF}'| sed -e 's/\.CH[0-9]\.wav/_REAL/' > dt05_real_${enhan}_wav.ids
+  if $eval_flag; then
+    if [ ! -f "${extra_annotations}/et05_real_2ch_track.list" ]; then
+      echo "Error: No such file: ${extra_annotations}/et05_real_2ch_track.list"
+      exit 1
+    fi
+    awk -v dir="${audio_dir}" '{ch1=dir "/" $1; ch2=dir "/" $2; print("sox -M " ch1 " " ch2 " -c 2 -t wav - |")}' ${extra_annotations}/et05_real_2ch_track.list > et05_real_$enhan.flist
+    awk -v dir="${audio_dir}" '{print(dir "/" $1)}' ${extra_annotations}/et05_real_2ch_track.list | sed -E 's#isolated_2ch_track/(.*)\.CH[0-9]\.wav#isolated_6ch_track/\1.CH0.wav#g' > et05_real_${enhan}_spk1.flist
+    awk -v dir="${audio_dir}" '{print(dir "/" $1)}' ${extra_annotations}/et05_real_2ch_track.list | awk -F'[/]' '{print $NF}'| sed -e 's/\.CH[0-9]\.wav/_REAL/' > et05_real_${enhan}_wav.ids
+  fi
+
+  # make a scp file from file list
+  for x in $list_set; do
+    paste -d" " ${x}_wav.ids $x.flist | sort -k 1 > ${x}_wav.scp
+    paste -d" " ${x}_wav.ids ${x}_spk1.flist | sort -k 1 > ${x}_spk1_wav.scp
+  done
+
 elif [[ "$track" == "6" ]]; then
   # 6-ch track
   for ch in $(seq 1 6); do
@@ -80,12 +113,14 @@ elif [[ "$track" == "6" ]]; then
 fi
 
 #make a transcription from dot
-cat tr05_real.dot | sed -e 's/(\(.*\))/\1/' | awk '{print $NF "_REAL"}'> tr05_real_$enhan.ids
-cat tr05_real.dot | sed -e 's/(.*)//' > tr05_real_$enhan.txt
-paste -d" " tr05_real_$enhan.ids tr05_real_$enhan.txt | sort -k 1 > tr05_real_$enhan.trans1
-if [[ "$track" != "6" ]]; then
-  mv tr05_real_$enhan.trans1 tr05_real_$enhan.trans1.6ch
-  awk 'FNR==NR{k=$1; $1=""; a[k]=$0; next}{split($0,lst,"."); print $1, a[lst[1] "_REAL"]}' tr05_real_$enhan.trans1.6ch tr05_real_${enhan}_wav.scp > tr05_real_$enhan.trans1
+if [[ "$track" != "2" ]]; then
+  cat tr05_real.dot | sed -e 's/(\(.*\))/\1/' | awk '{print $NF "_REAL"}'> tr05_real_$enhan.ids
+  cat tr05_real.dot | sed -e 's/(.*)//' > tr05_real_$enhan.txt
+  paste -d" " tr05_real_$enhan.ids tr05_real_$enhan.txt | sort -k 1 > tr05_real_$enhan.trans1
+  if [[ "$track" != "6" ]]; then
+    mv tr05_real_$enhan.trans1 tr05_real_$enhan.trans1.6ch
+    awk 'FNR==NR{k=$1; $1=""; a[k]=$0; next}{split($0,lst,"."); print $1, a[lst[1] "_REAL"]}' tr05_real_$enhan.trans1.6ch tr05_real_${enhan}_wav.scp > tr05_real_$enhan.trans1
+  fi
 fi
 cat dt05_real.dot | sed -e 's/(\(.*\))/\1/' | awk '{print $NF "_REAL"}'> dt05_real_$enhan.ids
 cat dt05_real.dot | sed -e 's/(.*)//' > dt05_real_$enhan.txt

egs2/chime4/enh1/local/simu_ext_chime4_data_prep.sh

@@ -15,8 +15,8 @@ if [ $# -ne 2 ]; then
   exit 1;
 fi
 
-if [[ ! " 1 6 " =~ " $track " ]]; then
-  echo "Error: \$track must be either 1 or 6: ${track}"
+if [[ ! " 1 2 6 " =~ " $track " ]]; then
+  echo "Error: \$track must be one of (1, 2, 6): ${track}"
   exit 1;
 fi
 
@@ -31,10 +31,16 @@ local=`pwd`/local
 utils=`pwd`/utils
 odir=`pwd`/data
 
+if [[ "$track" == "2" ]]; then
+  # no training data for 2-ch track; use training data in 6-ch track instead
+  list_set=""
+else
+  list_set="tr05_simu_$enhan"
+fi
 if $eval_flag; then
-list_set="tr05_simu_$enhan dt05_simu_$enhan et05_simu_$enhan"
+  list_set+=" dt05_simu_$enhan et05_simu_$enhan"
 else
-list_set="tr05_simu_$enhan dt05_simu_$enhan"
+  list_set+=" dt05_simu_$enhan"
 fi
 
 cd $dir
@@ -67,6 +73,30 @@ if [[ "$track" == "1" ]]; then
     sed -E "s#\.Clean\.wav#\.Noise\.wav#g" ${x}_spk1_wav.scp > ${x}_noise_wav.scp
   done
 
+elif [[ "$track" == "2" ]]; then
+  # 2-ch track
+  if [ ! -f "${annotations}/dt05_simu_2ch_track.list" ]; then
+    echo "Error: No such file: ${annotations}/dt05_simu_2ch_track.list"
+    exit 1
+  fi
+  awk -v dir="${audio_dir}/isolated" '{ch1=dir "/" $1; ch2=dir "/" $2; print("sox -M " ch1 " " ch2 " -c 2 -t wav - |")}' ${annotations}/dt05_simu_2ch_track.list > dt05_simu_$enhan.flist
+  awk -v dir="${audio_dir}/isolated" '{print(dir "/" $1)}' ${annotations}/dt05_simu_2ch_track.list | awk -F'[/]' '{print $NF}'| sed -e 's/\.CH[0-9]\.wav/_SIMU/' > dt05_simu_${enhan}_wav.ids
+  if $eval_flag; then
+    if [ ! -f "${extra_annotations}/et05_simu_2ch_track.list" ]; then
+      echo "Error: No such file: ${extra_annotations}/et05_simu_2ch_track.list"
+      exit 1
+    fi
+    awk -v dir="${audio_dir}/isolated" '{ch1=dir "/" $1; ch2=dir "/" $2; print("sox -M " ch1 " " ch2 " -c 2 -t wav - |")}' ${extra_annotations}/et05_simu_2ch_track.list > et05_simu_$enhan.flist
+    awk -v dir="${audio_dir}/isolated" '{print(dir "/" $1)}' ${extra_annotations}/et05_simu_2ch_track.list | awk -F'[/]' '{print $NF}'| sed -e 's/\.CH[0-9]\.wav/_SIMU/' > et05_simu_${enhan}_wav.ids
+  fi
+
+  # make a scp file from file list
+  for x in $list_set; do
+    paste -d" " ${x}_wav.ids $x.flist | sort -k 1 > ${x}_wav.scp
+    sed -E "s#${audio_dir}/isolated/(\S+\.CH[0-9]).wav#${audio_dir}/isolated_ext/\1.Clean.wav#g" ${x}_wav.scp > ${x}_spk1_wav.scp
+    sed -E "s#\.Clean\.wav#\.Noise\.wav#g" ${x}_spk1_wav.scp > ${x}_noise_wav.scp
+  done
+
 elif [[ "$track" == "6" ]]; then
   # 6-ch track
   for ch in $(seq 1 6); do
@@ -100,10 +130,12 @@ if [ ! -e dot_files.flist ]; then
   echo "Could not find $dir/dot_files.flist files, first run local/clean_wsj0_data_prep.sh";
   exit 1;
 fi
-cat tr05_simu_${enhan}_wav.scp | awk -F'[_]' '{print $2}' | tr '[A-Z]' '[a-z]' \
-    | $local/find_noisy_transcripts.pl dot_files.flist | cut -f 2- -d" " > tr05_simu_$enhan.txt
-cat tr05_simu_${enhan}_wav.scp | cut -f 1 -d" " > tr05_simu_$enhan.ids
-paste -d" " tr05_simu_$enhan.ids tr05_simu_$enhan.txt | sort -k 1 > tr05_simu_$enhan.trans1
+if [[ "$track" != "2" ]]; then
+  cat tr05_simu_${enhan}_wav.scp | awk -F'[_]' '{print $2}' | tr '[A-Z]' '[a-z]' \
+      | $local/find_noisy_transcripts.pl dot_files.flist | cut -f 2- -d" " > tr05_simu_$enhan.txt
+  cat tr05_simu_${enhan}_wav.scp | cut -f 1 -d" " > tr05_simu_$enhan.ids
+  paste -d" " tr05_simu_$enhan.ids tr05_simu_$enhan.txt | sort -k 1 > tr05_simu_$enhan.trans1
+fi
 # dt05 and et05 simulation data are generated from the CHiME4 booth recording
 # and we use CHiME4 dot files
 cat dt05_simu.dot | sed -e 's/(\(.*\))/\1/' | awk '{print $NF "_SIMU"}'> dt05_simu_$enhan.ids

egs2/librimix/enh1/local/asr_data_prep.sh

@@ -0,0 +1 @@
+../../asr1/local/data_prep.sh

egs2/librimix/enh1/local/data.sh

@@ -150,4 +150,31 @@ if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
     done
 fi
 
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+    log "stage 4: LibriMix Transcripts Files Prepration."
+
+    mkdir -p data/local
+    for dset in "train-clean-100" "train-clean-360" "dev-clean" "test-clean"; do
+        local/asr_data_prep.sh ${LIBRISPEECH}/${dset} data/local/${dset}
+        cp data/local/${dset}/text data/local/${dset}.txt
+    done
+    cat data/local/train-clean-100.txt data/local/train-clean-360.txt > data/local/train_clean_460.txt
+
+    for dset in "dev" "test" "train" "train-100" "train-360"; do
+        if [[ " ${dset} " =~ train* ]]; then
+            src_ls_dset_txt="train_clean_460.txt"
+        else
+            src_ls_dset_txt="${dset}-clean.txt"
+        fi
+
+        for i in $(seq ${num_spk}); do
+            awk -v idx=${i} '(FILENAME==ARGV[1]) {text[$1]=$0} (FILENAME==ARGV[2]) {split($1, lst, "_"); uttname=lst[idx]; print($1, text[uttname])}' \
+                data/local/${src_ls_dset_txt} data/${dset}/wav.scp | \
+                cut -d" " -f1,3- > data/${dset}/text_spk${i}
+        done
+    done
+
+    rm -rf data/local
+fi
+
 log "Successfully finished. [elapsed=${SECONDS}s]"

egs2/librimix/tse1/README.md

@@ -16,7 +16,7 @@
 
 |dataset|PESQ_WB|STOI|SAR|SDR|SIR|SI_SNR|OVRL|SIG|BAK|P808_MOS|
 |---|---|---|---|---|---|---|---|---|---|---|
-|dev|1.08|64.43|7.18|-1.71|0.08|-1.81|1.60|2.26|1.62|2.68|
-|test|1.08|64.56|6.90|-1.83|0.09|-1.93|1.63|2.33|1.66|2.71|
-|enhanced_dev|1.73|86.50|12.50|11.40|24.83|10.58|2.95|3.24|3.92|3.23|
-|enhanced_test|1.73|87.36|12.34|11.47|24.51|10.74|2.99|3.29|3.91|3.25|
+|dev_min_16k|1.08|64.43|7.18|-1.71|0.08|-1.81|1.60|2.26|1.62|2.68|
+|test_min_16k|1.08|64.56|6.90|-1.83|0.09|-1.93|1.63|2.33|1.66|2.71|
+|enhanced_dev_min_16k|1.73|86.50|12.50|11.40|24.83|10.58|2.95|3.24|3.92|3.23|
+|enhanced_test_min_16k|1.73|87.36|12.34|11.47|24.51|10.74|2.99|3.29|3.91|3.25|

espnet2/bin/enh_scoring.py

@@ -56,6 +56,7 @@ def scoring(
     inf_scp: List[str],
     ref_channel: int,
     flexible_numspk: bool,
+    is_tse: bool,
     use_dnsmos: bool,
     dnsmos_args: Dict,
     use_pesq: bool,
@@ -192,7 +193,9 @@ def scoring(
                 else:
                     num_spk = ref.shape[0]
 
-            sdr, sir, sar, perm = bss_eval_sources(ref, inf, compute_permutation=True)
+            sdr, sir, sar, perm = bss_eval_sources(
+                ref, inf, compute_permutation=not is_tse
+            )
 
             for i in range(num_spk):
                 stoi_score = stoi(ref[i], inf[int(perm[i])], fs_sig=sample_rate)
@@ -296,6 +299,7 @@ def get_parser():
     group.add_argument("--key_file", type=str)
     group.add_argument("--ref_channel", type=int, default=0)
     group.add_argument("--flexible_numspk", type=str2bool, default=False)
+    group.add_argument("--is_tse", type=str2bool, default=False)
 
     group = parser.add_argument_group("DNSMOS related")
     group.add_argument("--use_dnsmos", type=str2bool, default=False)

test/espnet2/bin/test_enh_scoring.py

@@ -26,7 +26,8 @@ def spk_scp(tmp_path):
 
 
 @pytest.mark.parametrize("flexible_numspk", [True, False])
-def test_scoring(tmp_path, spk_scp, flexible_numspk):
+@pytest.mark.parametrize("is_tse", [True, False])
+def test_scoring(tmp_path, spk_scp, flexible_numspk, is_tse):
     scoring(
         output_dir=str(tmp_path / "output"),
         dtype="float32",
@@ -36,6 +37,7 @@ def test_scoring(tmp_path, spk_scp, flexible_numspk):
         inf_scp=[spk_scp],
         ref_channel=0,
         flexible_numspk=flexible_numspk,
+        is_tse=is_tse,
         use_dnsmos=False,
         dnsmos_args={
             "mode": "local",