Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

data load for inference #788

Merged
merged 5 commits into from
Mar 30, 2018
Merged

data load for inference #788

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3bbb356
data load for inference
zhxfl Mar 28, 2018
5a09151
load batch name list
zhxfl Mar 28, 2018
f0160b2
roll back
zhxfl Mar 30, 2018
606cb22
Merge remote-tracking branch 'upstream/develop' into fix-661
zhxfl Mar 30, 2018
b0c1fc1
merge develop
zhxfl Mar 30, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 69 additions & 44 deletions fluid/DeepASR/data_utils/async_data_reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,12 @@ class SampleInfo(object):
label_bin_path (str): File containing the label data.
label_size (int): Byte count of the sample's label data.
label_frame_num (int): Label number of the sample.
sample_name (str): Key of the sample
"""

def __init__(self, feature_bin_path, feature_start, feature_size,
feature_frame_num, feature_dim, label_bin_path, label_start,
label_size, label_frame_num):
label_size, label_frame_num, sample_name):
self.feature_bin_path = feature_bin_path
self.feature_start = feature_start
self.feature_size = feature_size
Expand All @@ -45,6 +46,7 @@ def __init__(self, feature_bin_path, feature_start, feature_size,
self.label_start = label_start
self.label_size = label_size
self.label_frame_num = label_frame_num
self.sample_name = sample_name


class SampleInfoBucket(object):
Expand Down Expand Up @@ -102,24 +104,33 @@ def generate_sample_info_list(self):
feature_bin_path = self._feature_bin_paths[block_idx]
feature_desc_path = self._feature_desc_paths[block_idx]

label_desc_lines = open(label_desc_path).readlines()
feature_desc_lines = open(feature_desc_path).readlines()

sample_num = int(label_desc_lines[0].split()[1])
assert sample_num == int(feature_desc_lines[0].split()[1])
label_desc_lines = []
if label_desc_path != "":
label_desc_lines = open(label_desc_path).readlines()
sample_num = int(feature_desc_lines[0].split()[1])

if label_desc_path != "":
assert sample_num == int(label_desc_lines[0].split()[1])

for i in xrange(sample_num):
feature_desc_split = feature_desc_lines[i + 1].split()
sample_name = feature_desc_split[0]
feature_start = int(feature_desc_split[2])
feature_size = int(feature_desc_split[3])
feature_frame_num = int(feature_desc_split[4])
feature_dim = int(feature_desc_split[5])

label_desc_split = label_desc_lines[i + 1].split()
label_start = int(label_desc_split[2])
label_size = int(label_desc_split[3])
label_frame_num = int(label_desc_split[4])
assert feature_frame_num == label_frame_num
label_start = -1
label_size = -1
label_frame_num = feature_frame_num
if label_desc_path != "":
label_desc_split = label_desc_lines[i + 1].split()
label_start = int(label_desc_split[2])
label_size = int(label_desc_split[3])
label_frame_num = int(label_desc_split[4])
assert feature_frame_num == label_frame_num

if self._split_sentence_threshold == -1 or \
self._split_perturb == -1 or \
Expand All @@ -129,7 +140,7 @@ def generate_sample_info_list(self):
SampleInfo(feature_bin_path, feature_start,
feature_size, feature_frame_num, feature_dim,
label_bin_path, label_start, label_size,
label_frame_num))
label_frame_num, sample_name))
#split sentence
else:
cur_frame_pos = 0
Expand All @@ -150,13 +161,12 @@ def generate_sample_info_list(self):
* feature_dim * 4, cur_frame_len * feature_dim *
4, cur_frame_len, feature_dim, label_bin_path,
label_start + cur_frame_pos * 4, cur_frame_len *
4, cur_frame_len))
4, cur_frame_len, sample_name))

remain_frame_num -= cur_frame_len
cur_frame_pos += cur_frame_len
if remain_frame_num <= 0:
break

return sample_info_list


Expand Down Expand Up @@ -192,7 +202,7 @@ class AsyncDataReader(object):

def __init__(self,
feature_file_list,
label_file_list,
label_file_list="",
drop_frame_len=512,
proc_num=10,
sample_buffer_size=1024,
Expand Down Expand Up @@ -221,16 +231,24 @@ def __init__(self,
def generate_bucket_list(self, is_shuffle):
if self._block_info_list is None:
block_feature_info_lines = open(self._feature_file_list).readlines()
block_label_info_lines = open(self._label_file_list).readlines()
assert len(block_feature_info_lines) == len(block_label_info_lines)
self._block_info_list = []
for i in xrange(0, len(block_feature_info_lines), 2):
block_info = (block_feature_info_lines[i],
block_feature_info_lines[i + 1],
block_label_info_lines[i],
block_label_info_lines[i + 1])
self._block_info_list.append(
map(lambda line: line.strip(), block_info))
if self._label_file_list != "":
block_label_info_lines = open(self._label_file_list).readlines()
assert len(block_feature_info_lines) == len(
block_label_info_lines)
for i in xrange(0, len(block_feature_info_lines), 2):
block_info = (block_feature_info_lines[i],
block_feature_info_lines[i + 1],
block_label_info_lines[i],
block_label_info_lines[i + 1])
self._block_info_list.append(
map(lambda line: line.strip(), block_info))
else:
for i in xrange(0, len(block_feature_info_lines), 2):
block_info = (block_feature_info_lines[i],
block_feature_info_lines[i + 1], "", "")
self._block_info_list.append(
map(lambda line: line.strip(), block_info))

if is_shuffle:
self._rng.shuffle(self._block_info_list)
Expand Down Expand Up @@ -310,19 +328,25 @@ def read_bytes(fpath, start, size):
sample_info.feature_dim,
len(feature_bytes))

label_bytes = read_bytes(sample_info.label_bin_path,
sample_info.label_start,
sample_info.label_size)

assert sample_info.label_frame_num * 4 == len(label_bytes), (
sample_info.label_bin_path, sample_info.label_array,
len(label_bytes))

label_array = struct.unpack('I' * sample_info.label_frame_num,
label_bytes)
label_data = np.array(
label_array, dtype='int64').reshape(
(sample_info.label_frame_num, 1))
label_data = None
if sample_info.label_bin_path != "":
label_bytes = read_bytes(sample_info.label_bin_path,
sample_info.label_start,
sample_info.label_size)

assert sample_info.label_frame_num * 4 == len(
label_bytes), (sample_info.label_bin_path,
sample_info.label_array,
len(label_bytes))

label_array = struct.unpack(
'I' * sample_info.label_frame_num, label_bytes)
label_data = np.array(
label_array, dtype='int64').reshape(
(sample_info.label_frame_num, 1))
else:
label_data = np.zeros(
(sample_info.label_frame_num, 1), dtype='int64')

feature_frame_num = sample_info.feature_frame_num
feature_dim = sample_info.feature_dim
Expand All @@ -332,12 +356,11 @@ def read_bytes(fpath, start, size):
feature_data = np.array(
feature_array, dtype='float32').reshape((
sample_info.feature_frame_num, sample_info.feature_dim))

sample_data = (feature_data, label_data)
sample_data = (feature_data, label_data,
sample_info.sample_name)
for transformer in self._transformers:
# @TODO(pkuyym) to make transfomer only accept feature_data
sample_data = transformer.perform_trans(sample_data)

while order_id != out_order[0]:
time.sleep(0.001)

Expand Down Expand Up @@ -387,12 +410,14 @@ def batch_to_ndarray(batch_samples, lod):
batch_feature = np.zeros((lod[-1], frame_dim), dtype="float32")
batch_label = np.zeros((lod[-1], 1), dtype="int64")
start = 0
name_lst = []
for sample in batch_samples:
frame_num = sample[0].shape[0]
batch_feature[start:start + frame_num, :] = sample[0]
batch_label[start:start + frame_num, :] = sample[1]
start += frame_num
return (batch_feature, batch_label)
name_lst.append(sample[2])
return (batch_feature, batch_label, name_lst)

@suppress_complaints(verbose=self._verbose, notify=self._force_exit)
def batch_assembling_task(sample_generator, batch_queue):
Expand All @@ -402,16 +427,16 @@ def batch_assembling_task(sample_generator, batch_queue):
batch_samples.append(sample)
lod.append(lod[-1] + sample[0].shape[0])
if len(batch_samples) == batch_size:
(batch_feature, batch_label) = batch_to_ndarray(
(batch_feature, batch_label, name_lst) = batch_to_ndarray(
batch_samples, lod)
batch_queue.put((batch_feature, batch_label, lod))
batch_queue.put((batch_feature, batch_label, lod, name_lst))
batch_samples = []
lod = [0]

if len(batch_samples) >= minimum_batch_size:
(batch_feature, batch_label) = batch_to_ndarray(batch_samples,
lod)
batch_queue.put((batch_feature, batch_label, lod))
(batch_feature, batch_label, name_lst) = batch_to_ndarray(
batch_samples, lod)
batch_queue.put((batch_feature, batch_label, lod, name_lst))

batch_queue.put(EpochEndSignal())

Expand Down
6 changes: 3 additions & 3 deletions fluid/DeepASR/data_utils/augmentor/tests/test_data_trans.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def test(self):
feature = np.zeros((2, 120), dtype="float32")
feature.fill(1)
trans = trans_mean_variance_norm.TransMeanVarianceNorm(self._file_path)
(feature1, label1) = trans.perform_trans((feature, None))
(feature1, label1, name) = trans.perform_trans((feature, None, None))
(mean, var) = trans.get_mean_var()
feature_flat1 = feature1.flatten()
feature_flat = feature.flatten()
Expand Down Expand Up @@ -70,7 +70,7 @@ def test_perform(self):
feature[2, 0:40].fill(3)
feature[3, 0:40].fill(4)
trans = trans_add_delta.TransAddDelta()
(feature, label) = trans.perform_trans((feature, None))
(feature, label, name) = trans.perform_trans((feature, None, None))
self.assertAlmostEqual(feature.shape[0], 4)
self.assertAlmostEqual(feature.shape[1], 120)
self.assertAlmostEqual(1.0, feature[0][0])
Expand All @@ -93,7 +93,7 @@ def test_perfrom(self):
feature[i, :].fill(i)

trans = trans_splice.TransSplice()
(feature, label) = trans.perform_trans((feature, None))
(feature, label, name) = trans.perform_trans((feature, None, None))
self.assertEqual(feature.shape[1], 110)

for i in xrange(8):
Expand Down
6 changes: 3 additions & 3 deletions fluid/DeepASR/data_utils/augmentor/trans_add_delta.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ def perform_trans(self, sample):
Args:
sample(object,tuple): contain feature numpy and label numpy
Returns:
(feature, label)
(feature, label, name)
"""
(feature, label) = sample
(feature, label, name) = sample
frame_dim = feature.shape[1]
d_frame_dim = frame_dim * 3
head_filled = 5
Expand Down Expand Up @@ -64,7 +64,7 @@ def perform_trans(self, sample):
start * d_frame_dim + 2 * frame_dim, frame_dim, nframe,
d_frame_dim)
mat.shape = tmp_shape
return (mat[head_filled:mat.shape[0] - tail_filled, :], label)
return (mat[head_filled:mat.shape[0] - tail_filled, :], label, name)

def _regress(self, data_in, start_in, data_out, start_out, size, n, step):
""" regress
Expand Down
6 changes: 3 additions & 3 deletions fluid/DeepASR/data_utils/augmentor/trans_mean_variance_norm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,9 @@ def perform_trans(self, sample):
Args:
sample(object):input sample, contain feature numpy and label numpy
Returns:
(feature, label)
(feature, label, name)
"""
(feature, label) = sample
(feature, label, name) = sample
shape = feature.shape
assert len(shape) == 2
nfeature_len = shape[0] * shape[1]
Expand All @@ -68,4 +68,4 @@ def perform_trans(self, sample):
feature[ncur_idx:ncur_idx + self._nLen] = block
ncur_idx += self._nLen
feature = feature.reshape(shape)
return (feature, label)
return (feature, label, name)
6 changes: 3 additions & 3 deletions fluid/DeepASR/data_utils/augmentor/trans_splice.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ def perform_trans(self, sample):
Args:
sample(object): input sample(feature, label)
Return:
(feature, label)
(feature, label, name)
"""
(feature, label) = sample
(feature, label, name) = sample
nframe_num = feature.shape[0]
nframe_dim = feature.shape[1]
nnew_frame_dim = nframe_dim * (
Expand Down Expand Up @@ -61,4 +61,4 @@ def perform_trans(self, sample):
np.copyto(ret[i * nnew_frame_dim:(i + 1) * nnew_frame_dim],
mat[i * nframe_dim:i * nframe_dim + nnew_frame_dim])
ret = ret.reshape((nframe_num, nnew_frame_dim))
return (ret, label)
return (ret, label, name)
3 changes: 2 additions & 1 deletion fluid/DeepASR/train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ def test(exe):
# train data reader
train_data_reader = reader.AsyncDataReader(args.train_feature_lst,
args.train_label_lst, -1)

train_data_reader.set_transformers(ltrans)
# train
for pass_id in xrange(args.pass_num):
Expand All @@ -218,7 +219,7 @@ def test(exe):
train_data_reader.batch_iterator(args.batch_size,
args.minimum_batch_size)):
# load_data
(features, labels, lod) = batch_data
(features, labels, lod, name_lst) = batch_data
feature_t.set(features, place)
feature_t.set_lod([lod])
label_t.set(labels, place)
Expand Down