Skip to content
Permalink
Browse files

Sample loader : back to serial one core loader

  • Loading branch information
iperov committed Jan 11, 2020
1 parent bbe81b2 commit 52a67a61b317a4060b9f1142533e9916ee34f25d
Showing with 14 additions and 93 deletions.
  1. +14 −93 samplelib/SampleHost.py
@@ -70,25 +70,21 @@ def load(sample_type, samples_path):

@staticmethod
def load_face_samples ( image_paths):
result = FaceSamplesLoaderSubprocessor(image_paths).run()
sample_list = []

for filename, \
( face_type,
shape,
landmarks,
ie_polys,
eyebrows_expand_mod,
source_filename,
) in result:
sample_list.append( Sample(filename=filename,
sample_type=SampleType.FACE,
face_type=FaceType.fromString (face_type),
shape=shape,
landmarks=landmarks,
ie_polys=ie_polys,
eyebrows_expand_mod=eyebrows_expand_mod,
source_filename=source_filename,

for filename in io.progress_bar_generator (image_paths, desc="Loading"):
dflimg = DFLIMG.load (Path(filename))
if dflimg is None:
io.log_err (f"{filename} is not a dfl image file.")
else:
sample_list.append( Sample(filename=filename,
sample_type=SampleType.FACE,
face_type=FaceType.fromString ( dflimg.get_face_type() ),
shape=dflimg.get_shape(),
landmarks=dflimg.get_landmarks(),
ie_polys=dflimg.get_ie_polys(),
eyebrows_expand_mod=dflimg.get_eyebrows_expand_mod(),
source_filename=dflimg.get_source_filename(),
))
return sample_list

@@ -98,78 +94,3 @@ def upgradeToFaceTemporalSortedSamples( samples ):
new_s = sorted(new_s, key=operator.itemgetter(1))

return [ s[0] for s in new_s]


class FaceSamplesLoaderSubprocessor(Subprocessor):
#override
def __init__(self, image_paths ):
self.image_paths = image_paths
self.image_paths_len = len(image_paths)
self.idxs = [*range(self.image_paths_len)]
self.result = [None]*self.image_paths_len
super().__init__('FaceSamplesLoader', FaceSamplesLoaderSubprocessor.Cli, 60, initialize_subprocesses_in_serial=False)

#override
def on_clients_initialized(self):
io.progress_bar ("Loading", len (self.image_paths))

#override
def on_clients_finalized(self):
io.progress_bar_close()

#override
def process_info_generator(self):
for i in range(min(multiprocessing.cpu_count(), 8) ):
yield 'CPU%d' % (i), {}, {'device_idx': i,
'device_name': 'CPU%d' % (i),
}

#override
def get_data(self, host_dict):
if len (self.idxs) > 0:
idx = self.idxs.pop(0)
return idx, self.image_paths[idx]

return None

#override
def on_data_return (self, host_dict, data):
self.idxs.insert(0, data[0])

#override
def on_result (self, host_dict, data, result):
idx, dflimg = result
self.result[idx] = (self.image_paths[idx], dflimg)
io.progress_bar_inc(1)

#override
def get_result(self):
return self.result

class Cli(Subprocessor.Cli):
#override
def on_initialize(self, client_dict):
pass

#override
def process_data(self, data):
idx, filename = data
dflimg = DFLIMG.load (Path(filename))

if dflimg is None:
self.log_err (f"FaceSamplesLoader: {filename} is not a dfl image file.")
data = None
else:
data = (dflimg.get_face_type(),
dflimg.get_shape(),
dflimg.get_landmarks(),
dflimg.get_ie_polys(),
dflimg.get_eyebrows_expand_mod(),
dflimg.get_source_filename() )

return idx, data

#override
def get_data_name (self, data):
#return string identificator of your data
return data[1]

0 comments on commit 52a67a6

Please sign in to comment.
You can’t perform that action at this time.