Skip to content

Commit

Permalink
WARNING, TIMIT BASELINES CHANGE BY THIS COMMIT!
Browse files Browse the repository at this point in the history 
The change is in how the PER is computed. Previously there was optional silence with no output symbol, and forgiving of silence deletions. This was however slight cheating (Traditionally in TIMIT, the silence is scored).

The change consists of:
- removing the optional silence from he lexicon (see call of utils/prepare_lang.sh in run.sh)
- removing the forgiving of 'sil' deletions from scoring (see change in local/score.sh)
- updating the RESULTS

From now on the PER scores will involve no cheating and will be comparable to experiments in the literature.



git-svn-id: https://svn.code.sf.net/p/kaldi/code/trunk@4639 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8
  • Loading branch information
@KarelVesely84
KarelVesely84 committed Nov 20, 2014
1 parent f2ceaa7 commit bdd752b
Show file tree
Hide file tree
Showing 4 changed files with 47 additions and 50 deletions.
72 changes: 36 additions & 36 deletions egs/timit/s5/RESULTS
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash
for x in exp/*/decode*; do [ -d $x ] && echo $x | grep "${1:-.*}" >/dev/null && grep WER $x/wer_* 2>/dev/null | utils/best_wer.sh; done
for x in exp/*/decode*; do [ -d $x ] && echo $x | grep "${1:-.*}" >/dev/null && grep Sum $x/score_*/*.sys 2>/dev/null | utils/best_wer.sh; done
for x in exp/{mono,tri,sgmm,dnn,combine}*/decode*; do [ -d $x ] && echo $x | grep "${1:-.*}" >/dev/null && grep WER $x/wer_* 2>/dev/null | utils/best_wer.sh; done
for x in exp/{mono,tri,sgmm,dnn,combine}*/decode*; do [ -d $x ] && echo $x | grep "${1:-.*}" >/dev/null && grep Sum $x/score_*/*.sys 2>/dev/null | utils/best_wer.sh; done
exit 0

# Use caution when comparing these results with other published results.
Expand All @@ -13,82 +13,82 @@ Phone mapping : Training with 48 phonemes, for testing mapped to 39 phonemes

# monophone, deltas.
---------------------------------Dev Set------------------------------------------
%WER 29.6 | 400 15057 | 73.3 19.8 6.9 2.9 29.6 100.0 | -1.345 | exp/mono/decode_dev/score_3/ctm_39phn.filt.sys
%WER 31.7 | 400 15057 | 71.8 19.5 8.7 3.5 31.7 100.0 | -0.457 | exp/mono/decode_dev/score_5/ctm_39phn.filt.sys
--------------------------------Test Set------------------------------------------
%WER 30.2 | 192 7215 | 72.1 19.5 8.4 2.3 30.2 100.0 | -0.760 | exp/mono/decode_test/score_4/ctm_39phn.filt.sys
%WER 32.7 | 192 7215 | 70.5 19.8 9.6 3.2 32.7 100.0 | -0.482 | exp/mono/decode_test/score_5/ctm_39phn.filt.sys


# tri1 : first triphone system (delta+delta-delta features)
---------------------------------Dev Set------------------------------------------
%WER 22.7 | 400 15057 | 79.6 15.0 5.4 2.3 22.7 99.5 | -0.110 | exp/tri1/decode_dev/score_10/ctm_39phn.filt.sys
%WER 25.1 | 400 15057 | 78.9 15.9 5.2 4.0 25.1 99.8 | -0.178 | exp/tri1/decode_dev/score_10/ctm_39phn.filt.sys
--------------------------------Test Set------------------------------------------
%WER 24.4 | 192 7215 | 78.3 16.5 5.2 2.7 24.4 100.0 | -0.207 | exp/tri1/decode_test/score_9/ctm_39phn.filt.sys
%WER 25.6 | 192 7215 | 78.3 15.9 5.8 3.9 25.6 100.0 | -0.129 | exp/tri1/decode_test/score_10/ctm_39phn.filt.sys


# tri2 : an LDA+MLLT system
---------------------------------Dev Set------------------------------------------
%WER 21.3 | 400 15057 | 81.1 14.0 4.9 2.4 21.3 99.8 | -0.266 | exp/tri2/decode_dev/score_9/ctm_39phn.filt.sys
%WER 23.0 | 400 15057 | 80.7 14.6 4.7 3.7 23.0 99.5 | -0.230 | exp/tri2/decode_dev/score_10/ctm_39phn.filt.sys
--------------------------------Test Set------------------------------------------
%WER 22.6 | 192 7215 | 79.7 14.9 5.4 2.3 22.6 99.5 | -0.260 | exp/tri2/decode_test/score_10/ctm_39phn.filt.sys
%WER 23.7 | 192 7215 | 80.0 14.8 5.2 3.7 23.7 99.5 | -0.284 | exp/tri2/decode_test/score_10/ctm_39phn.filt.sys


# tri3 : Speaker Adaptive Training (SAT) system
---------------------------------Dev Set------------------------------------------
%WER 18.8 | 400 15057 | 83.5 12.7 3.9 2.3 18.8 99.5 | -0.771 | exp/tri3/decode_dev/score_8/ctm_39phn.filt.sys
%WER 20.3 | 400 15057 | 82.7 12.8 4.5 3.1 20.3 99.8 | -0.556 | exp/tri3/decode_dev/score_10/ctm_39phn.filt.sys
--------------------------------Test Set------------------------------------------
%WER 19.8 | 192 7215 | 82.2 13.1 4.8 2.0 19.8 99.0 | -0.545 | exp/tri3/decode_test/score_10/ctm_39phn.filt.sys
%WER 21.6 | 192 7215 | 81.6 13.6 4.9 3.2 21.6 99.5 | -0.560 | exp/tri3/decode_test/score_10/ctm_39phn.filt.sys


# SGMM2 Training :
---------------------------------Dev Set------------------------------------------
%WER 16.5 | 400 15057 | 85.8 11.0 3.2 2.3 16.5 98.8 | -0.599 | exp/sgmm2_4/decode_dev/score_6/ctm_39phn.filt.sys
%WER 17.8 | 400 15057 | 85.1 11.0 3.9 2.9 17.8 99.3 | -0.451 | exp/sgmm2_4/decode_dev/score_7/ctm_39phn.filt.sys
--------------------------------Test Set------------------------------------------
%WER 17.8 | 192 7215 | 84.3 11.9 3.9 2.1 17.8 99.0 | -0.438 | exp/sgmm2_4/decode_test/score_7/ctm_39phn.filt.sys
%WER 19.7 | 192 7215 | 83.2 12.2 4.6 3.0 19.7 99.0 | -0.291 | exp/sgmm2_4/decode_test/score_8/ctm_39phn.filt.sys


# SGMM2 + MMI Training :
---------------------------------Dev Set------------------------------------------
%WER 16.7 | 400 15057 | 85.5 11.1 3.3 2.2 16.7 99.0 | -0.392 | exp/sgmm2_4_mmi_b0.1/decode_dev_it1/score_7/ctm_39phn.filt.sys
%WER 16.7 | 400 15057 | 85.6 11.2 3.3 2.3 16.7 99.0 | -0.423 | exp/sgmm2_4_mmi_b0.1/decode_dev_it2/score_7/ctm_39phn.filt.sys
%WER 17.0 | 400 15057 | 85.5 11.2 3.2 2.5 17.0 99.5 | -0.442 | exp/sgmm2_4_mmi_b0.1/decode_dev_it3/score_7/ctm_39phn.filt.sys
%WER 16.9 | 400 15057 | 85.6 11.2 3.2 2.5 16.9 99.5 | -0.474 | exp/sgmm2_4_mmi_b0.1/decode_dev_it4/score_7/ctm_39phn.filt.sys
%WER 18.0 | 400 15057 | 85.6 11.2 3.3 3.6 18.0 98.8 | -0.599 | exp/sgmm2_4_mmi_b0.1/decode_dev_it1/score_6/ctm_39phn.filt.sys
%WER 18.0 | 400 15057 | 85.7 11.2 3.1 3.6 18.0 99.0 | -0.619 | exp/sgmm2_4_mmi_b0.1/decode_dev_it2/score_6/ctm_39phn.filt.sys
%WER 18.1 | 400 15057 | 85.6 11.3 3.1 3.7 18.1 98.8 | -0.646 | exp/sgmm2_4_mmi_b0.1/decode_dev_it3/score_6/ctm_39phn.filt.sys
%WER 18.1 | 400 15057 | 85.3 11.3 3.4 3.4 18.1 99.0 | -0.463 | exp/sgmm2_4_mmi_b0.1/decode_dev_it4/score_7/ctm_39phn.filt.sys
--------------------------------Test Set------------------------------------------
%WER 18.2 | 192 7215 | 84.3 12.1 3.6 2.6 18.2 98.4 | -0.443 | exp/sgmm2_4_mmi_b0.1/decode_test_it1/score_7/ctm_39phn.filt.sys
%WER 18.2 | 192 7215 | 84.1 12.0 3.9 2.4 18.2 99.0 | -0.299 | exp/sgmm2_4_mmi_b0.1/decode_test_it2/score_8/ctm_39phn.filt.sys
%WER 18.4 | 192 7215 | 84.3 12.2 3.5 2.7 18.4 99.0 | -0.441 | exp/sgmm2_4_mmi_b0.1/decode_test_it3/score_7/ctm_39phn.filt.sys
%WER 18.3 | 192 7215 | 84.4 12.1 3.5 2.7 18.3 98.4 | -0.461 | exp/sgmm2_4_mmi_b0.1/decode_test_it4/score_7/ctm_39phn.filt.sys
%WER 19.9 | 192 7215 | 83.4 12.3 4.3 3.4 19.9 99.5 | -0.300 | exp/sgmm2_4_mmi_b0.1/decode_test_it1/score_8/ctm_39phn.filt.sys
%WER 20.2 | 192 7215 | 83.0 12.3 4.6 3.2 20.2 99.0 | -0.208 | exp/sgmm2_4_mmi_b0.1/decode_test_it2/score_9/ctm_39phn.filt.sys
%WER 20.2 | 192 7215 | 83.4 12.4 4.2 3.7 20.2 99.5 | -0.333 | exp/sgmm2_4_mmi_b0.1/decode_test_it3/score_8/ctm_39phn.filt.sys
%WER 20.3 | 192 7215 | 83.0 12.6 4.5 3.3 20.3 99.0 | -0.235 | exp/sgmm2_4_mmi_b0.1/decode_test_it4/score_9/ctm_39phn.filt.sys
# bMMI not helpful here...

# Hybrid System (Dans DNN):
---------------------------------Dev Set------------------------------------------
%WER 19.0 | 400 15057 | 83.6 12.6 3.8 2.6 19.0 99.8 | -1.186 | exp/tri4_nnet/decode_dev/score_3/ctm_39phn.filt.sys
%WER 21.1 | 400 15057 | 81.9 12.6 5.6 3.0 21.1 99.5 | -0.485 | exp/tri4_nnet/decode_dev/score_5/ctm_39phn.filt.sys
--------------------------------Test Set------------------------------------------
%WER 20.5 | 192 7215 | 82.0 13.3 4.8 2.4 20.5 99.5 | -0.696 | exp/tri4_nnet/decode_test/score_4/ctm_39phn.filt.sys
%WER 23.0 | 192 7215 | 79.4 13.5 7.1 2.4 23.0 100.0 | -0.138 | exp/tri4_nnet/decode_test/score_7/ctm_39phn.filt.sys

# Hybrid System (Karel's DNN)
---------------------------------Dev Set------------------------------------------
%WER 15.8 | 400 15057 | 86.2 10.4 3.4 2.0 15.8 99.0 | -1.040 | exp/dnn4_pretrain-dbn_dnn/decode_dev/score_4/ctm_39phn.filt.sys
%WER 17.5 | 400 15057 | 84.6 10.5 4.8 2.2 17.5 98.5 | -0.471 | exp/dnn4_pretrain-dbn_dnn/decode_dev/score_6/ctm_39phn.filt.sys
--------------------------------Test Set------------------------------------------
%WER 17.3 | 192 7215 | 85.0 11.2 3.8 2.2 17.3 99.0 | -1.140 | exp/dnn4_pretrain-dbn_dnn/decode_test/score_4/ctm_39phn.filt.sys
%WER 18.5 | 192 7215 | 84.2 11.0 4.8 2.7 18.5 100.0 | -1.151 | exp/dnn4_pretrain-dbn_dnn/decode_test/score_4/ctm_39phn.filt.sys
# Hybrid System (Karel's DNN), sMBR training
---------------------------------Dev Set------------------------------------------
%WER 15.9 | 400 15057 | 86.5 10.6 3.0 2.3 15.9 99.3 | -1.053 | exp/dnn4_pretrain-dbn_dnn_smbr/decode_dev_it1/score_4/ctm_39phn.filt.sys
%WER 16.1 | 400 15057 | 86.6 10.7 2.7 2.7 16.1 99.3 | -0.724 | exp/dnn4_pretrain-dbn_dnn_smbr/decode_dev_it6/score_5/ctm_39phn.filt.sys
%WER 17.3 | 400 15057 | 85.5 10.6 4.0 2.7 17.3 98.5 | -0.696 | exp/dnn4_pretrain-dbn_dnn_smbr/decode_dev_it1/score_5/ctm_39phn.filt.sys
%WER 17.3 | 400 15057 | 85.4 10.7 3.9 2.7 17.3 98.5 | -0.380 | exp/dnn4_pretrain-dbn_dnn_smbr/decode_dev_it6/score_7/ctm_39phn.filt.sys
--------------------------------Test Set------------------------------------------
%WER 17.6 | 192 7215 | 85.1 11.4 3.5 2.7 17.6 99.5 | -1.161 | exp/dnn4_pretrain-dbn_dnn_smbr/decode_test_it1/score_4/ctm_39phn.filt.sys
%WER 17.8 | 192 7215 | 85.2 11.5 3.3 3.0 17.8 99.5 | -0.871 | exp/dnn4_pretrain-dbn_dnn_smbr/decode_test_it6/score_5/ctm_39phn.filt.sys
%WER 18.6 | 192 7215 | 84.2 11.1 4.7 2.8 18.6 100.0 | -0.816 | exp/dnn4_pretrain-dbn_dnn_smbr/decode_test_it1/score_5/ctm_39phn.filt.sys
%WER 18.8 | 192 7215 | 84.7 11.4 3.9 3.5 18.8 100.0 | -0.819 | exp/dnn4_pretrain-dbn_dnn_smbr/decode_test_it6/score_5/ctm_39phn.filt.sys
# sMBR not helpful here...


# Combination SGMM + Dans DNN:
---------------------------------Dev Set------------------------------------------
%WER 16.6 | 400 15057 | 85.5 11.1 3.4 2.1 16.6 99.5 | -0.089 | exp/combine_2/decode_dev_it1/score_6/ctm_39phn.filt.sys
%WER 16.6 | 400 15057 | 85.2 11.0 3.7 1.9 16.6 99.3 | 0.045 | exp/combine_2/decode_dev_it2/score_8/ctm_39phn.filt.sys
%WER 16.7 | 400 15057 | 85.9 11.0 3.1 2.6 16.7 99.5 | -0.092 | exp/combine_2/decode_dev_it3/score_6/ctm_39phn.filt.sys
%WER 16.7 | 400 15057 | 86.0 11.0 3.0 2.7 16.7 99.5 | -0.091 | exp/combine_2/decode_dev_it4/score_6/ctm_39phn.filt.sys
%WER 16.7 | 400 15057 | 86.0 10.9 3.1 2.7 16.7 99.5 | -0.102 | exp/combine_2/decode_dev_it1/score_6/ctm_39phn.filt.sys
%WER 16.7 | 400 15057 | 86.4 10.8 2.8 3.1 16.7 99.5 | -0.248 | exp/combine_2/decode_dev_it2/score_5/ctm_39phn.filt.sys
%WER 16.8 | 400 15057 | 85.8 10.9 3.3 2.6 16.8 99.3 | -0.013 | exp/combine_2/decode_dev_it3/score_7/ctm_39phn.filt.sys
%WER 16.9 | 400 15057 | 86.2 11.0 2.8 3.1 16.9 99.8 | -0.240 | exp/combine_2/decode_dev_it4/score_5/ctm_39phn.filt.sys
--------------------------------Test Set------------------------------------------
%WER 17.8 | 192 7215 | 84.7 11.7 3.6 2.4 17.8 99.0 | -0.107 | exp/combine_2/decode_test_it1/score_6/ctm_39phn.filt.sys
%WER 18.0 | 192 7215 | 84.8 11.9 3.4 2.8 18.0 99.0 | -0.113 | exp/combine_2/decode_test_it2/score_6/ctm_39phn.filt.sys
%WER 18.0 | 192 7215 | 84.9 11.8 3.3 2.9 18.0 99.5 | -0.125 | exp/combine_2/decode_test_it3/score_6/ctm_39phn.filt.sys
%WER 18.0 | 192 7215 | 84.7 11.8 3.5 2.7 18.0 99.0 | -0.017 | exp/combine_2/decode_test_it4/score_7/ctm_39phn.filt.sys
%WER 18.4 | 192 7215 | 84.6 12.0 3.5 3.0 18.4 99.0 | -0.223 | exp/combine_2/decode_test_it1/score_5/ctm_39phn.filt.sys
%WER 18.5 | 192 7215 | 84.5 12.1 3.4 3.0 18.5 99.0 | -0.215 | exp/combine_2/decode_test_it2/score_5/ctm_39phn.filt.sys
%WER 18.5 | 192 7215 | 84.4 12.0 3.7 2.9 18.5 99.0 | -0.074 | exp/combine_2/decode_test_it3/score_6/ctm_39phn.filt.sys
%WER 18.6 | 192 7215 | 84.9 12.0 3.1 3.6 18.6 99.0 | -0.451 | exp/combine_2/decode_test_it4/score_4/ctm_39phn.filt.sys

9 changes: 5 additions & 4 deletions egs/timit/s5/cmd.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,11 @@


#b) BUT cluster options
export train_cmd="queue.pl -q all.q@blade[01][0126789][123456789] -l ram_free=2500M,mem_free=2500M,matylda5=0.5"
export decode_cmd="queue.pl -q all.q@blade[01][0126789][123456789] -l ram_free=3000M,mem_free=3000M,matylda5=0.1"
export mkgraph_cmd="queue.pl -q all.q@blade[01][0126789][123456789] -l ram_free=4G,mem_free=4G,matylda5=3"
export cuda_cmd="queue.pl -q long.q@pcspeech-gpu,long.q@dellgpu*,long.q@pco203-0[0124] -l gpu=1"
queue="all.q@@blade,all.q@@speech,all.q@dellgpu*,all.q@supergpu*"
export train_cmd="queue.pl -q $queue -l ram_free=2500M,mem_free=2500M,matylda5=0.5"
export decode_cmd="queue.pl -q $queue -l ram_free=3000M,mem_free=3000M,matylda5=0.1"
export mkgraph_cmd="queue.pl -q $queue -l ram_free=4G,mem_free=4G,matylda5=3"
export cuda_cmd="queue.pl -q long.q@pcspeech-gpu,long.q@dellgpu1,long.q@pcgpu*,long.q@supergpu1 -l gpu=1"

#c) run locally...
#export train_cmd=run.pl
Expand Down
3 changes: 1 addition & 2 deletions egs/timit/s5/local/score_sclite.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,7 @@ done
mkdir -p $dir/scoring/log

# Map reference to 39 phone classes, the silence is optional (.):
local/timit_norm_trans.pl -i $data/stm -m $phonemap -from 48 -to 39 | \
sed 's: sil: (sil):g' > $dir/scoring/stm_39phn
local/timit_norm_trans.pl -i $data/stm -m $phonemap -from 48 -to 39 >$dir/scoring/stm_39phn
cp $data/glm $dir/scoring/glm_39phn

if [ $stage -le 0 ]; then
Expand Down
13 changes: 5 additions & 8 deletions egs/timit/s5/run.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,18 +42,15 @@ local/timit_data_prep.sh $timit || exit 1

local/timit_prepare_dict.sh

# Caution below: we insert optional-silence with probability 0.5, which is the
# default, but this is probably not appropriate for this setup, since silence
# appears also as a word in the dictionary and is scored. We could stop this
# by using the option --sil-prob 0.0, but apparently this makes results worse.
# (-> In sclite scoring the deletions of 'sil' are not scored as errors)
utils/prepare_lang.sh --position-dependent-phones false --num-sil-states 3 \
# Caution below: we remove optional silence by setting "--sil-prob 0.0",
# in TIMIT the silence appears also as a word in the dictionary and is scored.
utils/prepare_lang.sh --sil-prob 0.0 --position-dependent-phones false --num-sil-states 3 \
data/local/dict "sil" data/local/lang_tmp data/lang

local/timit_format_data.sh

echo ============================================================================
echo " MFCC Feature Extration & CMVN for Training and Test set "
echo " MFCC Feature Extration & CMVN for Training and Test set "
echo ============================================================================

# Now make MFCC features.
Expand Down Expand Up @@ -144,7 +141,7 @@ echo ===========================================================================
steps/align_fmllr.sh --nj "$train_nj" --cmd "$train_cmd" \
data/train data/lang exp/tri3 exp/tri3_ali

#exit 0 # From this point you can run DNN : local/run_dnn.sh
#exit 0 # From this point you can run Karel's DNN : local/run_dnn.sh

steps/train_ubm.sh --cmd "$train_cmd" \
$numGaussUBM data/train data/lang exp/tri3_ali exp/ubm4
Expand Down

0 comments on commit bdd752b

Please sign in to comment.