-
Notifications
You must be signed in to change notification settings - Fork 1
/
parallel-kaldi-latgen-biglm-local
executable file
·209 lines (196 loc) · 9.15 KB
/
parallel-kaldi-latgen-biglm-local
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
#!/bin/bash
set -e;
export LC_NUMERIC=C;
### Script directory.
SDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
### Load useful functions.
. "${SDIR}/parallel-kaldi-functions.inc.sh" || ( \
echo "ERROR: Sourcing file \"${SDIR}/parallel-kaldi-functions.inc.sh\"">&2 \
&& exit 1);
# Options for gmm-latgen-biglm-faster
acoustic_scale=1.0;
allow_partial=false;
beam=1000;
beam_delta=0.5;
delta=0.000976562;
determinize_lattice=true;
gzip=true;
hash_ratio=2;
lattice_beam=10;
log_sum_exp_prune=0;
max_active=2147483647;
max_mem=50000000;
min_active=200;
minimize=false;
phone_determinize=true;
prune_interval=25;
word_determinize=true;
word_symbol_table="";
write_best_words=false;
write_best_align=false;
# Generic options
overwrite=false;
help_message="
Usage: parallel-kaldi-latgen-biglm-local [options] mdl fst old_lm new_lm
fea1 [fea2 ...] work_dir
Description:
Generate lattices from a GMM-based model and a decoding FST, using a
separate big language model FST. Optionally, the 1-best word decoding
and alignment can also be obtained.
The input decoding FST, the old language model FST, the new language
model FST and the features are interpreted as rxspecifiers (see Kaldi
documentation).
e.g: parallel-kaldi-latgen-biglm-local mdl hclg.fst old_G.fst new_G.fst \\
ark:test.fea wdir
Options:
--acoustic_scale : (float, default = $acoustic_scale)
Scaling factor for acoustic likelihoods.
--allow_partial : (boolean, default = $allow_partial)
If true, produce output even if end state was
not reached.
--beam : (float, default = $beam)
Decoding beam.
--beam_delta : (float, default = $beam_delta)
Increment used in decoding. Larger is more
accurate (this parameter is obscure and relates
to a speedup in the way the max-active
constraint is applied).
--config : (string, default = \"\")
Read options from this config file. Multiple
config files can be specified.
--delta : (float, default = $delta)
Tolerance used in determinization.
--determinize_lattice : (boolean, default = $determinize_lattice)
If true, determinize the lattice (in a special
sense, keeping only best pdf-sequence for each
word-sequence).
--gzip : (boolean, default = $gzip)
If true, compress the output files using gzip.
--hash_ratio : (float, default = $hash_ratio)
Setting used in decoder to control hash
behavior.
--lattice_beam : (float, default = $lattice_beam)
Lattice generation beam.
--log_sum_exp_prune : (float, default = $log_sum_exp_prune)
If >0, pruning parameter to minimize exp()'s.
Suggest 3 to 5; larger is more exact.
--max_active : (integer, default = $max_active)
Decoder maximum number of active states.
--max_mem : (integer, default = $max_mem)
Maximum approximate memory usage in
determinization (real usage might be many times
this).
--min_active : (integer, default = $min_active)
Decoder minimum number of active states.
--minimize : (boolean, default = $minimize)
If true, push and minimize after
determinization.
--overwrite : (boolean, default = $overwrite)
If true, overwrite previously created files.
--phone_determinize : (boolean, default = $phone_determinize)
If true, do an initial pass of determinization
on both phones and words.
--prune_interval : (integer, default = $prune_interval)
Interval (in frames) at which to prune tokens.
--word_determinize : (boolean, default = $word_determinize)
If true, do a second pass of determinization on
words only (see also --phone_determinize).
--word_symbol_table : (string, default = \"$word_symbol_table\")
Symbol table for words (for debug output).
--write_best_align : (boolean, default = $write_best_align)
If true, writes an ARK file with the 1-best
alignment hypothesis in the output directory.
--write_best_words : (boolean, default = $write_best_words)
If true, writes an ARK file with the 1-best
word sequence hypothesis in the output
directory.
";
. "${SDIR}/parallel-kaldi-cmd-parse.inc.sh" || \
error "Sourcing file \"${SDIR}/parallel-kaldi-cmd-parse.inc.sh\"";
[ $# -eq 0 ] && echo "$help_message" >&2 && exit 1;
[ $# -lt 6 ] && error "Wrong number of arguments. Check the usage with --help";
mdl="$1";
fst="$2";
old_lm="$3";
new_lm="$4";
shift 4;
feas=();
while [ $# -gt 1 ]; do feas+=("$1"); shift; done;
wdir="$1";
check_execs bc gmm-latgen-biglm-faster;
check_files "$mdl";
make_dirs "$wdir";
[ -z "$word_symbol_table" ] || check_files "$word_symbol_table";
# Check option values
[ "$(echo "$acoustic_scale >= 0.0" | bc -l)" = 1 ] || \
error "--acoustic_scale must be greater than or equal to 0.0!";
[ "$(echo "$beam > 0.0" | bc -l)" = 1 ] || \
error "--beam must be greater than 0.0!";
[ "$(echo "$beam_delta > 0.0" | bc -l)" = 1 ] || \
error "--beam_delta must be greater than 0.0!";
[ "$(echo "$delta > 0.0" | bc -l)" = 1 ] || \
error "--delta must be greater than 0.0!";
[ "$hash_ratio" -gt 1 ] || \
error "--hash_ratio must be a positive integer greater than 1!";
[ "$(echo "$lattice_beam > 0.0" | bc -l)" = 1 ] || \
error "--lattice_beam must be greater than 0.0!";
[ "$(echo "$log_sum_exp_prune >= 0.0" | bc -l)" = 1 ] || \
error "--log_sum_exp_prune must be greater than or equal to 0.0!";
[ "$max_active" -gt 0 ] || \
error "--max_active must be a positive integer greater than 0!";
[ "$min_active" -ge 0 ] || \
error "--min_active must be a positive integer greater than or equal to 0!";
[ "$min_active" -le "$max_active" ] || \
error "--min_active must be lower than or equal to --max_active!";
[ "$prune_interval" -gt 0 ] || \
error "--prune_interval must be a positive integer greater than 0!";
# If best alignment is requested, 1-best word hypothesis has to be written too.
[ "$write_best_align" = true ] && write_best_words=true;
# Process all input features in parallel
bg_jobs=();
for i in $(seq 1 ${#feas[@]}); do
olattice="$wdir/lattice.$[i-1].ark";
owords="$wdir/words.$[i-1].ark";
oalign="$wdir/align.$[i-1].ark";
if [ "$gzip" = true ]; then
olattice_spec="ark:|gzip -c -9 > $olattice.gz";
owords_spec="ark:|gzip -c -9 > $owords.gz";
oalign_spec="ark:|gzip -c -9 > $oalign.gz";
else
olattice_spec="ark:$olattice";
owords_spec="ark:$owords";
oalign_spec="ark:$oalign";
fi;
[[ "$overwrite" = false && -s "$olattice" && \
( "$write_best_words" = false || -s "$owords" ) && \
( "$write_best_align" = false || -s "$oalign" ) ]] || {
gmm-latgen-biglm-faster \
--acoustic-scale="$acoustic_scale" \
--allow-partial="$allow_partial" --beam="$beam" \
--beam-delta="$beam_delta" --delta="$delta" \
--determinize-lattice="$determinize_lattice" \
--hash-ratio="$hash_ratio" --lattice-beam="$lattice_beam" \
--log-sum-exp-prune="$log_sum_exp_prune" \
--max-active="$max_active" --max-mem="$max_mem" \
--min-active="$min_active" --minimize="$minimize" \
--phone-determinize="$phone_determinize" \
--prune-interval="$prune_interval" \
--word-determinize="$word_determinize" \
--word-symbol-table="$word_symbol_table" \
"$mdl" "$fst" "$old_lm" "$new_lm" "${feas[i-1]}" "$olattice_spec" \
$([ "$write_best_words" = true ] && echo "$owords_spec") \
$([ "$write_best_align" = true ] && echo "$oalign_spec") \
&> "$wdir/gmm-latgen-biglm-faster.$[i-1].log" &
bg_jobs+=("$!");
}
done;
# Wait for all processes running in the background.
any_failure=0;
for i in $(seq 1 ${#bg_jobs[@]}); do
log="$wdir/gmm-latgen-biglm-faster.$[i-1].log";
wait ${bg_jobs[i-1]} || ( \
error_continue "gmm-latgen-biglm-faster failed, see log file \"$log\"" && \
any_failure=1 );
done;
[ $any_failure -eq 0 ] || exit 1;
exit 0;