Skip to content

Commit

Permalink
Merge pull request #788 from zhxfl/fix-661
Browse files Browse the repository at this point in the history 
data load for inference
  • Loading branch information
Yibing Liu committed Mar 30, 2018
2 parents 218d199 + b0c1fc1 commit 9e1459d
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 57 deletions.
113 changes: 69 additions & 44 deletions fluid/DeepASR/data_utils/async_data_reader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,12 @@ class SampleInfo(object):
label_bin_path (str): File containing the label data.
label_size (int): Byte count of the sample's label data.
label_frame_num (int): Label number of the sample.
sample_name (str): Key of the sample
"""

def __init__(self, feature_bin_path, feature_start, feature_size,
feature_frame_num, feature_dim, label_bin_path, label_start,
label_size, label_frame_num):
label_size, label_frame_num, sample_name):
self.feature_bin_path = feature_bin_path
self.feature_start = feature_start
self.feature_size = feature_size
Expand All @@ -45,6 +46,7 @@ def __init__(self, feature_bin_path, feature_start, feature_size,
self.label_start = label_start
self.label_size = label_size
self.label_frame_num = label_frame_num
self.sample_name = sample_name


class SampleInfoBucket(object):
Expand Down Expand Up @@ -102,24 +104,33 @@ def generate_sample_info_list(self):
feature_bin_path = self._feature_bin_paths[block_idx]
feature_desc_path = self._feature_desc_paths[block_idx]

label_desc_lines = open(label_desc_path).readlines()
feature_desc_lines = open(feature_desc_path).readlines()

sample_num = int(label_desc_lines[0].split()[1])
assert sample_num == int(feature_desc_lines[0].split()[1])
label_desc_lines = []
if label_desc_path != "":
label_desc_lines = open(label_desc_path).readlines()
sample_num = int(feature_desc_lines[0].split()[1])

if label_desc_path != "":
assert sample_num == int(label_desc_lines[0].split()[1])

for i in xrange(sample_num):
feature_desc_split = feature_desc_lines[i + 1].split()
sample_name = feature_desc_split[0]
feature_start = int(feature_desc_split[2])
feature_size = int(feature_desc_split[3])
feature_frame_num = int(feature_desc_split[4])
feature_dim = int(feature_desc_split[5])

label_desc_split = label_desc_lines[i + 1].split()
label_start = int(label_desc_split[2])
label_size = int(label_desc_split[3])
label_frame_num = int(label_desc_split[4])
assert feature_frame_num == label_frame_num
label_start = -1
label_size = -1
label_frame_num = feature_frame_num
if label_desc_path != "":
label_desc_split = label_desc_lines[i + 1].split()
label_start = int(label_desc_split[2])
label_size = int(label_desc_split[3])
label_frame_num = int(label_desc_split[4])
assert feature_frame_num == label_frame_num

if self._split_sentence_threshold == -1 or \
self._split_perturb == -1 or \
Expand All @@ -129,7 +140,7 @@ def generate_sample_info_list(self):
SampleInfo(feature_bin_path, feature_start,
feature_size, feature_frame_num, feature_dim,
label_bin_path, label_start, label_size,
label_frame_num))
label_frame_num, sample_name))
#split sentence
else:
cur_frame_pos = 0
Expand All @@ -150,13 +161,12 @@ def generate_sample_info_list(self):
* feature_dim * 4, cur_frame_len * feature_dim *
4, cur_frame_len, feature_dim, label_bin_path,
label_start + cur_frame_pos * 4, cur_frame_len *
4, cur_frame_len))
4, cur_frame_len, sample_name))

remain_frame_num -= cur_frame_len
cur_frame_pos += cur_frame_len
if remain_frame_num <= 0:
break

return sample_info_list


Expand Down Expand Up @@ -192,7 +202,7 @@ class AsyncDataReader(object):

def __init__(self,
feature_file_list,
label_file_list,
label_file_list="",
drop_frame_len=512,
proc_num=10,
sample_buffer_size=1024,
Expand Down Expand Up @@ -221,16 +231,24 @@ def __init__(self,
def generate_bucket_list(self, is_shuffle):
if self._block_info_list is None:
block_feature_info_lines = open(self._feature_file_list).readlines()
block_label_info_lines = open(self._label_file_list).readlines()
assert len(block_feature_info_lines) == len(block_label_info_lines)
self._block_info_list = []
for i in xrange(0, len(block_feature_info_lines), 2):
block_info = (block_feature_info_lines[i],
block_feature_info_lines[i + 1],
block_label_info_lines[i],
block_label_info_lines[i + 1])
self._block_info_list.append(
map(lambda line: line.strip(), block_info))
if self._label_file_list != "":
block_label_info_lines = open(self._label_file_list).readlines()
assert len(block_feature_info_lines) == len(
block_label_info_lines)
for i in xrange(0, len(block_feature_info_lines), 2):
block_info = (block_feature_info_lines[i],
block_feature_info_lines[i + 1],
block_label_info_lines[i],
block_label_info_lines[i + 1])
self._block_info_list.append(
map(lambda line: line.strip(), block_info))
else:
for i in xrange(0, len(block_feature_info_lines), 2):
block_info = (block_feature_info_lines[i],
block_feature_info_lines[i + 1], "", "")
self._block_info_list.append(
map(lambda line: line.strip(), block_info))

if is_shuffle:
self._rng.shuffle(self._block_info_list)
Expand Down Expand Up @@ -310,19 +328,25 @@ def read_bytes(fpath, start, size):
sample_info.feature_dim,
len(feature_bytes))

label_bytes = read_bytes(sample_info.label_bin_path,
sample_info.label_start,
sample_info.label_size)

assert sample_info.label_frame_num * 4 == len(label_bytes), (
sample_info.label_bin_path, sample_info.label_array,
len(label_bytes))

label_array = struct.unpack('I' * sample_info.label_frame_num,
label_bytes)
label_data = np.array(
label_array, dtype='int64').reshape(
(sample_info.label_frame_num, 1))
label_data = None
if sample_info.label_bin_path != "":
label_bytes = read_bytes(sample_info.label_bin_path,
sample_info.label_start,
sample_info.label_size)

assert sample_info.label_frame_num * 4 == len(
label_bytes), (sample_info.label_bin_path,
sample_info.label_array,
len(label_bytes))

label_array = struct.unpack(
'I' * sample_info.label_frame_num, label_bytes)
label_data = np.array(
label_array, dtype='int64').reshape(
(sample_info.label_frame_num, 1))
else:
label_data = np.zeros(
(sample_info.label_frame_num, 1), dtype='int64')

feature_frame_num = sample_info.feature_frame_num
feature_dim = sample_info.feature_dim
Expand All @@ -332,12 +356,11 @@ def read_bytes(fpath, start, size):
feature_data = np.array(
feature_array, dtype='float32').reshape((
sample_info.feature_frame_num, sample_info.feature_dim))

sample_data = (feature_data, label_data)
sample_data = (feature_data, label_data,
sample_info.sample_name)
for transformer in self._transformers:
# @TODO(pkuyym) to make transfomer only accept feature_data
sample_data = transformer.perform_trans(sample_data)

while order_id != out_order[0]:
time.sleep(0.001)

Expand Down Expand Up @@ -387,12 +410,14 @@ def batch_to_ndarray(batch_samples, lod):
batch_feature = np.zeros((lod[-1], frame_dim), dtype="float32")
batch_label = np.zeros((lod[-1], 1), dtype="int64")
start = 0
name_lst = []
for sample in batch_samples:
frame_num = sample[0].shape[0]
batch_feature[start:start + frame_num, :] = sample[0]
batch_label[start:start + frame_num, :] = sample[1]
start += frame_num
return (batch_feature, batch_label)
name_lst.append(sample[2])
return (batch_feature, batch_label, name_lst)

@suppress_complaints(verbose=self._verbose, notify=self._force_exit)
def batch_assembling_task(sample_generator, batch_queue):
Expand All @@ -402,16 +427,16 @@ def batch_assembling_task(sample_generator, batch_queue):
batch_samples.append(sample)
lod.append(lod[-1] + sample[0].shape[0])
if len(batch_samples) == batch_size:
(batch_feature, batch_label) = batch_to_ndarray(
(batch_feature, batch_label, name_lst) = batch_to_ndarray(
batch_samples, lod)
batch_queue.put((batch_feature, batch_label, lod))
batch_queue.put((batch_feature, batch_label, lod, name_lst))
batch_samples = []
lod = [0]

if len(batch_samples) >= minimum_batch_size:
(batch_feature, batch_label) = batch_to_ndarray(batch_samples,
lod)
batch_queue.put((batch_feature, batch_label, lod))
(batch_feature, batch_label, name_lst) = batch_to_ndarray(
batch_samples, lod)
batch_queue.put((batch_feature, batch_label, lod, name_lst))

batch_queue.put(EpochEndSignal())

Expand Down
6 changes: 3 additions & 3 deletions fluid/DeepASR/data_utils/augmentor/tests/test_data_trans.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def test(self):
feature = np.zeros((2, 120), dtype="float32")
feature.fill(1)
trans = trans_mean_variance_norm.TransMeanVarianceNorm(self._file_path)
(feature1, label1) = trans.perform_trans((feature, None))
(feature1, label1, name) = trans.perform_trans((feature, None, None))
(mean, var) = trans.get_mean_var()
feature_flat1 = feature1.flatten()
feature_flat = feature.flatten()
Expand Down Expand Up @@ -70,7 +70,7 @@ def test_perform(self):
feature[2, 0:40].fill(3)
feature[3, 0:40].fill(4)
trans = trans_add_delta.TransAddDelta()
(feature, label) = trans.perform_trans((feature, None))
(feature, label, name) = trans.perform_trans((feature, None, None))
self.assertAlmostEqual(feature.shape[0], 4)
self.assertAlmostEqual(feature.shape[1], 120)
self.assertAlmostEqual(1.0, feature[0][0])
Expand All @@ -93,7 +93,7 @@ def test_perfrom(self):
feature[i, :].fill(i)

trans = trans_splice.TransSplice()
(feature, label) = trans.perform_trans((feature, None))
(feature, label, name) = trans.perform_trans((feature, None, None))
self.assertEqual(feature.shape[1], 110)

for i in xrange(8):
Expand Down
6 changes: 3 additions & 3 deletions fluid/DeepASR/data_utils/augmentor/trans_add_delta.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ def perform_trans(self, sample):
Args:
sample(object,tuple): contain feature numpy and label numpy
Returns:
(feature, label)
(feature, label, name)
"""
(feature, label) = sample
(feature, label, name) = sample
frame_dim = feature.shape[1]
d_frame_dim = frame_dim * 3
head_filled = 5
Expand Down Expand Up @@ -64,7 +64,7 @@ def perform_trans(self, sample):
start * d_frame_dim + 2 * frame_dim, frame_dim, nframe,
d_frame_dim)
mat.shape = tmp_shape
return (mat[head_filled:mat.shape[0] - tail_filled, :], label)
return (mat[head_filled:mat.shape[0] - tail_filled, :], label, name)

def _regress(self, data_in, start_in, data_out, start_out, size, n, step):
""" regress
Expand Down
6 changes: 3 additions & 3 deletions fluid/DeepASR/data_utils/augmentor/trans_mean_variance_norm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,9 @@ def perform_trans(self, sample):
Args:
sample(object):input sample, contain feature numpy and label numpy
Returns:
(feature, label)
(feature, label, name)
"""
(feature, label) = sample
(feature, label, name) = sample
shape = feature.shape
assert len(shape) == 2
nfeature_len = shape[0] * shape[1]
Expand All @@ -68,4 +68,4 @@ def perform_trans(self, sample):
feature[ncur_idx:ncur_idx + self._nLen] = block
ncur_idx += self._nLen
feature = feature.reshape(shape)
return (feature, label)
return (feature, label, name)
6 changes: 3 additions & 3 deletions fluid/DeepASR/data_utils/augmentor/trans_splice.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ def perform_trans(self, sample):
Args:
sample(object): input sample(feature, label)
Return:
(feature, label)
(feature, label, name)
"""
(feature, label) = sample
(feature, label, name) = sample
nframe_num = feature.shape[0]
nframe_dim = feature.shape[1]
nnew_frame_dim = nframe_dim * (
Expand Down Expand Up @@ -61,4 +61,4 @@ def perform_trans(self, sample):
np.copyto(ret[i * nnew_frame_dim:(i + 1) * nnew_frame_dim],
mat[i * nframe_dim:i * nframe_dim + nnew_frame_dim])
ret = ret.reshape((nframe_num, nnew_frame_dim))
return (ret, label)
return (ret, label, name)
3 changes: 2 additions & 1 deletion fluid/DeepASR/train.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ def test(exe):
# train data reader
train_data_reader = reader.AsyncDataReader(args.train_feature_lst,
args.train_label_lst, -1)

train_data_reader.set_transformers(ltrans)
# train
for pass_id in xrange(args.pass_num):
Expand All @@ -218,7 +219,7 @@ def test(exe):
train_data_reader.batch_iterator(args.batch_size,
args.minimum_batch_size)):
# load_data
(features, labels, lod) = batch_data
(features, labels, lod, name_lst) = batch_data
feature_t.set(features, place)
feature_t.set_lod([lod])
label_t.set(labels, place)
Expand Down

0 comments on commit 9e1459d

Please sign in to comment.