Cosmetic and other minor fixes for segmentation

egs/aspire/s5/local/multi_condition/get_ctm.sh

@@ -7,8 +7,7 @@ decode_mbr=true
 filter_ctm_command=cp
 glm=
 stm=
-window=10
-overlap=5
+resolve_overlaps=true
 [ -f ./path.sh ] && . ./path.sh
 . parse_options.sh || exit 1;
 
@@ -61,11 +60,16 @@ lattice-align-words-lexicon --output-error-lats=true --output-if-empty=true --ma
  $lang/phones/align_lexicon.int $model ark:- ark:- | \
 lattice-to-ctm-conf $frame_shift_opt --decode-mbr=$decode_mbr ark:- $decode_dir/score_$LMWT/penalty_$wip/ctm.overlapping || exit 1;
 
+ctm=$decode_dir/score_$LMWT/penalty_$wip/ctm.overlapping
 # combine the segment-wise ctm files, while resolving overlaps
-python local/multi_condition/resolve_ctm_overlaps.py --overlap $overlap --window-length $window $data_dir/utt2spk $decode_dir/score_$LMWT/penalty_$wip/ctm.overlapping $decode_dir/score_$LMWT/penalty_$wip/ctm.merged || exit 1;
-merged_ctm=$decode_dir/score_$LMWT/penalty_$wip/ctm.merged
+if $resolve_overlaps; then
+  utils/ctm/resolve_ctm_overlaps.py $data_dir/segments \
+    $decode_dir/score_$LMWT/penalty_$wip/ctm.overlapping \
+    $decode_dir/score_$LMWT/penalty_$wip/ctm.merged || exit 1;
+  ctm=$decode_dir/score_$LMWT/penalty_$wip/ctm.merged
+fi
 
-cat $merged_ctm | utils/int2sym.pl -f 5 $lang/words.txt | \
+cat $ctm | utils/int2sym.pl -f 5 $lang/words.txt | \
 utils/convert_ctm.pl $data_dir/segments $data_dir/reco2file_and_channel | \
 sort -k1,1 -k2,2 -k3,3nb > $decode_dir/score_$LMWT/penalty_$wip/ctm || exit 1;
 # Remove some stuff we don't want to score, from the ctm.

egs/aspire/s5/local/score_aspire.sh

@@ -14,10 +14,9 @@ word_ins_penalties=0.0,0.25,0.5,0.75,1.0
 default_wip=0.0
 ctm_beam=6
 decode_mbr=true
-window=30
-overlap=5
 cmd=run.pl
 stage=1
+resolve_overlaps=true
 tune_hyper=true # if true:
                 #    if the data set is "dev_aspire" we check for the
                 #       best lmwt and word_insertion_penalty,
@@ -89,7 +88,7 @@ if  $tune_hyper ; then
   # or use the default values
 
   if [ $stage -le 1 ]; then
-    if [ "$act_data_set" == "dev_aspire" ]; then
+    if [[ "$act_data_set" =~ "dev_aspire" ]]; then
       wip_string=$(echo $word_ins_penalties | sed 's/,/ /g')
       temp_wips=($wip_string)
       $cmd WIP=1:${#temp_wips[@]} $decode_dir/scoring/log/score.wip.WIP.log \
@@ -98,8 +97,8 @@ if  $tune_hyper ; then
         echo \$wip \&\& \
         $cmd LMWT=$min_lmwt:$max_lmwt $decode_dir/scoring/log/score.LMWT.\$wip.log \
           local/multi_condition/get_ctm.sh --filter-ctm-command "$filter_ctm_command" \
-            --window $window --overlap $overlap \
             --beam $ctm_beam --decode-mbr $decode_mbr \
+            --resolve-overlaps $resolve_overlaps \
             --glm data/${act_data_set}/glm --stm data/${act_data_set}/stm \
           LMWT \$wip $lang data/${segmented_data_set}_hires $model $decode_dir || exit 1;
 
@@ -124,7 +123,7 @@ wipfile.close()
   fi
 
 
-  if [ "$act_data_set" == "test_aspire" ] || [ "$act_data_set" == "eval_aspire" ]; then
+  if [[ "$act_data_set" =~ "test_aspire" ]] || [[ "$act_data_set" =~ "eval_aspire" ]]; then
     # check for the best values from dev_aspire decodes
     dev_decode_dir=$(echo $decode_dir|sed "s/test_aspire/dev_aspire_whole/g; s/eval_aspire/dev_aspire_whole/g")
     if [ -f $dev_decode_dir/scoring/bestLMWT ]; then

egs/aspire/s5/path.sh

@@ -2,4 +2,5 @@ export KALDI_ROOT=`pwd`/../../..
 export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$PWD:$PATH
 [ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1
 . $KALDI_ROOT/tools/config/common_path.sh
+export PATH=$KALDI_ROOT/tools/sctk/bin:$PATH
 export LC_ALL=C

egs/babel/s5d/cmd.sh

@@ -7,8 +7,8 @@
 
 #a) JHU cluster options
 export train_cmd="queue.pl -l arch=*64"
-export decode_cmd="queue.pl -l arch=*64,mem_free=2G,ram_free=2G"
-export mkgraph_cmd="queue.pl -l arch=*64,ram_free=4G,mem_free=4G"
+export decode_cmd="queue.pl --mem 4G"
+export mkgraph_cmd="queue.pl --mem 4G"
 
 #export cuda_cmd="..."
 

egs/babel/s5d/conf/common.limitedLP

@@ -38,7 +38,7 @@ use_pitch=true
 lmwt_plp_extra_opts=( --min-lmwt 8 --max-lmwt 12 )
 lmwt_bnf_extra_opts=( --min-lmwt 15 --max-lmwt 22 )
 lmwt_dnn_extra_opts=( --min-lmwt 10 --max-lmwt 15 )
-
+lmwt_chain_extra_opts=( --min-lmwt 8 --max-lmwt 12 )
 dnn_beam=16.0
 dnn_lat_beam=8.5
 

egs/babel/s5d/local/lattice_to_ctm.sh

@@ -10,6 +10,13 @@ word_ins_penalty=0.5
 min_lmwt=7
 max_lmwt=17
 model=
+resolve_overlaps=false   # If true, the words decoded in the regions where
+                         # two segments A and B are overlapping are resolved 
+                         # such that only words before the mid-point of the
+                         # overlapping region are taken to be hypothesized for 
+                         # segment A, and only words after the mid-point of 
+                         # the overlapping region are taken to be 
+                         # hypothesized for segment B.
 
 #end configuration section.
 
@@ -43,10 +50,22 @@ for f in $lang/words.txt $model $data/segments $data/reco2file_and_channel $dir/
   [ ! -f $f ] && echo "$0: expecting file $f to exist" && exit 1;
 done
 
+if [ -f $dir/../frame_subsampling_factor ]; then
+  factor=$(cat $dir/../frame_subsampling_factor) || exit 1
+  frame_shift_opt="--frame-shift=0.0$factor"
+  echo "$0: $dir/../frame_subsampling_factor exists, using $frame_shift_opt"
+fi
+
 name=`basename $data`; # e.g. eval2000
 
 mkdir -p $dir/scoring/log
 
+resolve_overlaps_cmd="cat"
+
+if $resolve_overlaps; then
+  resolve_overlaps_cmd="utils/ctm/resolve_ctm_overlaps.py $data/segments - -"
+fi
+
 if [ $stage -le 0 ]; then
   if [ ! -f $lang/phones/word_boundary.int ] ; then
     $cmd LMWT=$min_lmwt:$max_lmwt $dir/scoring/log/get_ctm.LMWT.log \
@@ -56,8 +75,9 @@ if [ $stage -le 0 ]; then
       lattice-add-penalty --word-ins-penalty=$word_ins_penalty ark:- ark:- \| \
       lattice-prune --beam=$beam ark:- ark:- \| \
       lattice-align-words-lexicon $lang/phones/align_lexicon.int $model ark:- ark:- \| \
-      lattice-to-ctm-conf --decode-mbr=$decode_mbr ark:- - \| \
+      lattice-to-ctm-conf $frame_shift_opt --decode-mbr=$decode_mbr ark:- - \| \
       utils/int2sym.pl -f 5 $lang/words.txt  \| tee $dir/score_LMWT/$name.utt.ctm \| \
+      $resolve_overlaps_cmd \| \
       utils/convert_ctm.pl $data/segments $data/reco2file_and_channel \
       '>' $dir/score_LMWT/$name.ctm || exit 1;
   else
@@ -68,8 +88,9 @@ if [ $stage -le 0 ]; then
       lattice-add-penalty --word-ins-penalty=$word_ins_penalty ark:- ark:- \| \
       lattice-prune --beam=$beam ark:- ark:- \| \
       lattice-align-words $lang/phones/word_boundary.int $model ark:- ark:- \| \
-      lattice-to-ctm-conf --decode-mbr=$decode_mbr ark:- - \| \
+      lattice-to-ctm-conf $frame_shift_opt --decode-mbr=$decode_mbr ark:- - \| \
       utils/int2sym.pl -f 5 $lang/words.txt  \| tee $dir/score_LMWT/$name.utt.ctm \| \
+      $resolve_overlaps_cmd \| \
       utils/convert_ctm.pl $data/segments $data/reco2file_and_channel \
       '>' $dir/score_LMWT/$name.ctm || exit 1;
   fi

egs/babel/s5d/local/run_kws_stt_task2.sh

@@ -32,6 +32,10 @@ extra_kws=false
 cmd=run.pl
 max_states=150000
 wip=0.5 #Word insertion penalty
+resolve_overlaps=false   # Set this to true, if there are overlapping segments
+                         # as input and the words in the CTM in the 
+                         # overlapping regions must be resolved to one 
+                         # of the segments.
 #End of options
 
 if [ $(basename $0) == score.sh ]; then
@@ -56,7 +60,7 @@ decode_dir=$3;
 if ! $skip_stt ; then
   if  [ ! -f $decode_dir/.score.done ] && [ ! -f $decode_dir/.done.score ]; then
     local/lattice_to_ctm.sh --cmd "$cmd" --word-ins-penalty $wip \
-      --min-lmwt ${min_lmwt} --max-lmwt ${max_lmwt} \
+      --min-lmwt ${min_lmwt} --max-lmwt ${max_lmwt} --resolve-overlaps $resolve_overlaps \
       $data_dir $lang_dir $decode_dir
 
     if ! $skip_scoring ; then

egs/babel/s5d/path.sh

@@ -1,7 +1,6 @@
-export KALDI_ROOT=/export/a09/jtrmal/kaldi/
+export KALDI_ROOT=../../..
 . $KALDI_ROOT/tools/env.sh
-. /export/a09/jtrmal/kaldi-current/tools/env.sh
-. /export/babel/data/software/env.sh 
+. /export/babel/data/software/env.sh
 export PATH=$PWD/utils/:$KALDI_ROOT/tools/sph2pipe_v2.5/:$KALDI_ROOT/src/bin:$KALDI_ROOT/tools/openfst/bin:$KALDI_ROOT/src/fstbin/:$KALDI_ROOT/src/gmmbin/:$KALDI_ROOT/src/featbin/:$KALDI_ROOT/src/lm/:$KALDI_ROOT/src/sgmmbin/:$KALDI_ROOT/src/sgmm2bin/:$KALDI_ROOT/src/fgmmbin/:$KALDI_ROOT/src/latbin/:$KALDI_ROOT/src/nnet3bin:$KALDI_ROOT/src/nnetbin:$KALDI_ROOT/src/nnet2bin/:$KALDI_ROOT/src/online2bin:$KALDI_ROOT/src/ivectorbin:$KALDI_ROOT/src/kwsbin:$PWD:$PATH
 export LC_ALL=C
 

egs/babel/s5d/run-4-anydecode.sh

@@ -232,13 +232,17 @@ if [ ! -f  $dataset_dir/.done ] ; then
     elif [ "$dataset_segments" == "train" ] ||\
          [ "$dataset_segments" == "pem" ]; then
       . ./local/datasets/supervised_pem.sh || exit 1
+    elif [[ $dataset_segments =~ seg* ]]; then
+      echo "Using ${dataset_dir} directly"
     else
       echo "Unknown type of the dataset: \"$dataset_segments\"!";
       echo "Valid dataset types are: seg, uem, pem";
       exit 1
     fi
   elif [ "$dataset_kind" == "unsupervised" ] ; then
-    if [ "$dataset_segments" == "seg" ] ; then
+    if [ "$dataset_segments" == "seg" ]; then
+      . ./local/datasets/unsupervised_seg.sh
+    elif [[ $dataset_segments =~ *seg* ]]; then
       . ./local/datasets/unsupervised_seg.sh
     elif [ "$dataset_segments" == "uem" ] ; then
       . ./local/datasets/unsupervised_uem.sh
@@ -251,6 +255,8 @@ if [ ! -f  $dataset_dir/.done ] ; then
       echo "does not really make any sense!"
       exit 1
       #. ./local/datasets/unsupervised_pem.sh
+    elif [[ $dataset_segments =~ seg* ]]; then
+      echo "Using ${dataset_dir} directly"
     else
       echo "Unknown type of the dataset: \"$dataset_segments\"!";
       echo "Valid dataset types are: seg, uem, pem";

egs/swbd/s5c/local/score_sclite.sh

@@ -38,10 +38,7 @@ for f in $data/stm $data/glm $lang/words.txt $lang/phones/word_boundary.int \
   [ ! -f $f ] && echo "$0: expecting file $f to exist" && exit 1;
 done
 
-if [ -f $dir/../frame_shift ]; then
-  frame_shift_opt="--frame-shift=$(cat $dir/../frame_shift)"
-  echo "$0: $dir/../frame_shift exists, using $frame_shift_opt"
-elif [ -f $dir/../frame_subsampling_factor ]; then
+if [ -f $dir/../frame_subsampling_factor ]; then
   factor=$(cat $dir/../frame_subsampling_factor) || exit 1
   frame_shift_opt="--frame-shift=0.0$factor"
   echo "$0: $dir/../frame_subsampling_factor exists, using $frame_shift_opt"

egs/wsj/s5/steps/compute_cmvn_stats.sh

@@ -91,18 +91,18 @@ if $fake; then
   ! cat $data/spk2utt | awk -v dim=$dim '{print $1, "["; for (n=0; n < dim; n++) { printf("0 "); } print "1";
                                                         for (n=0; n < dim; n++) { printf("1 "); } print "0 ]";}' | \
     copy-matrix ark:- ark,scp:$cmvndir/cmvn_$name.ark,$cmvndir/cmvn_$name.scp && \
-     echo "Error creating fake CMVN stats" && exit 1;
+     echo "Error creating fake CMVN stats.  See $logdir/cmvn_$name.log." && exit 1;
 elif $two_channel; then
   ! compute-cmvn-stats-two-channel $data/reco2file_and_channel scp:$data/feats.scp \
        ark,scp:$cmvndir/cmvn_$name.ark,$cmvndir/cmvn_$name.scp \
-    2> $logdir/cmvn_$name.log && echo "Error computing CMVN stats (using two-channel method)" && exit 1;
+    2> $logdir/cmvn_$name.log && echo "Error computing CMVN stats (using two-channel method). See $logdir/cmvn_$name.log." && exit 1;
 elif [ ! -z "$fake_dims" ]; then
   ! compute-cmvn-stats --spk2utt=ark:$data/spk2utt scp:$data/feats.scp ark:- | \
     modify-cmvn-stats "$fake_dims" ark:- ark,scp:$cmvndir/cmvn_$name.ark,$cmvndir/cmvn_$name.scp && \
-    echo "Error computing (partially fake) CMVN stats" && exit 1;
+    echo "Error computing (partially fake) CMVN stats.  See $logdir/cmvn_$name.log" && exit 1;
 else
   ! compute-cmvn-stats --spk2utt=ark:$data/spk2utt scp:$data/feats.scp ark,scp:$cmvndir/cmvn_$name.ark,$cmvndir/cmvn_$name.scp \
-    2> $logdir/cmvn_$name.log && echo "Error computing CMVN stats" && exit 1;
+    2> $logdir/cmvn_$name.log && echo "Error computing CMVN stats. See $logdir/cmvn_$name.log" && exit 1;
 fi
 
 cp $cmvndir/cmvn_$name.scp $data/cmvn.scp || exit 1;

egs/wsj/s5/steps/decode_nolats.sh

@@ -23,7 +23,7 @@ max_active=7000
 beam=13.0
 lattice_beam=6.0
 acwt=0.083333 # note: only really affects pruning (scoring is on lattices).
-write_alignments=false
+write_alignments=false  # The output directory is treated like an alignment directory
 write_words=true
 # End configuration section.
 
@@ -85,9 +85,29 @@ splice_opts=`cat $srcdir/splice_opts 2>/dev/null`
 cmvn_opts=`cat $srcdir/cmvn_opts 2>/dev/null`
 delta_opts=`cat $srcdir/delta_opts 2>/dev/null`
 
+utils/lang/check_phones_compatible.sh $graphdir/phones.txt $srcdir/phones.txt || exit 1;
+
+if $write_alignments; then
+  # Copy model and options that are generally expected in an alignment 
+  # directory.
+  cp $graphdir/phones.txt $dir || exit 1;
+
+  cp $srcdir/{tree,final.mdl} $dir || exit 1;
+  cp $srcdir/final.alimdl $dir 2>/dev/null
+  cp $srcdir/final.occs $dir;
+  cp $srcdir/splice_opts $dir 2>/dev/null # frame-splicing options.
+  cp $srcdir/cmvn_opts $dir 2>/dev/null # cmn/cmvn option.
+  cp $srcdir/delta_opts $dir 2>/dev/null
+fi
+
 case $feat_type in
   delta) feats="ark,s,cs:apply-cmvn $cmvn_opts --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp scp:$sdata/JOB/feats.scp ark:- | add-deltas $delta_opts ark:- ark:- |";;
-  lda) feats="ark,s,cs:apply-cmvn $cmvn_opts --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp scp:$sdata/JOB/feats.scp ark:- | splice-feats $splice_opts ark:- ark:- | transform-feats $srcdir/final.mat ark:- ark:- |";;
+  lda) feats="ark,s,cs:apply-cmvn $cmvn_opts --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp scp:$sdata/JOB/feats.scp ark:- | splice-feats $splice_opts ark:- ark:- | transform-feats $srcdir/final.mat ark:- ark:- |"
+    if $write_alignments; then
+      cp $srcdir/final.mat $dir
+      cp $srcdir/full.mat $dir 2>/dev/null
+    fi
+    ;;
   *) echo "Invalid feature type $feat_type" && exit 1;
 esac
 if [ ! -z "$transform_dir" ]; then # add transforms to features...