Skip to content

add new case for cached_batch_copy #783

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jantonguirao merged 6 commits into from
Apr 16, 2019
Merged

add new case for cached_batch_copy #783

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
221fcc9
Change version to 0.10.0dev
JanuszL Apr 12, 2019
6218bbb
Merge branch 'master' of https://github.com/kychennv/DALI
kychennv Apr 15, 2019
5b23fc6
Merge remote-tracking branch 'upstream/master'
kychennv Apr 16, 2019
c216d14
Update test_pipeline.py
kychennv Apr 16, 2019
be10e4d
update code statement
kychennv Apr 16, 2019
a15c771
update code statement
kychennv Apr 16, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 28 additions & 20 deletions dali/test/python/test_pipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1091,48 +1091,48 @@ def iter_setup(self):


class CachedPipeline(Pipeline):
def __init__(self, reader_type, batch_size, is_cached=False, seed=123456, skip_cached_images=False, num_shards=100000):
def __init__(self, reader_type, batch_size, is_cached=False, is_cached_batch_copy=True, seed=123456, skip_cached_images=False, num_shards=100000):
super(CachedPipeline, self).__init__(batch_size, num_threads=1, device_id=0, prefetch_queue_depth=1, seed=seed)
self.reader_type = reader_type
if reader_type == "MXNetReader":
self.input = ops.MXNetReader(path = "/data/imagenet/train-480-val-256-recordio/train.rec",
index_path = "/data/imagenet/train-480-val-256-recordio/train.idx",
shard_id = 0,
num_shards = num_shards,
shard_id = 0,
num_shards = num_shards,
stick_to_shard = True,
skip_cached_images = skip_cached_images,
skip_cached_images = skip_cached_images,
prefetch_queue_depth = 1)
elif reader_type == "CaffeReader":
self.input = ops.CaffeReader(path = "/data/imagenet/train-lmdb-256x256",
shard_id = 0,
num_shards = num_shards,
self.input = ops.CaffeReader(path = "/data/imagenet/train-lmdb-256x256",
shard_id = 0,
num_shards = num_shards,
stick_to_shard = True,
skip_cached_images = skip_cached_images,
skip_cached_images = skip_cached_images,
prefetch_queue_depth = 1)
elif reader_type == "Caffe2Reader":
self.input = ops.Caffe2Reader(path = "/data/imagenet/train-c2lmdb-480",
shard_id = 0,
num_shards = num_shards,
self.input = ops.Caffe2Reader(path = "/data/imagenet/train-c2lmdb-480",
shard_id = 0,
num_shards = num_shards,
stick_to_shard = True,
skip_cached_images = skip_cached_images,
skip_cached_images = skip_cached_images,
prefetch_queue_depth = 1)
elif reader_type == "FileReader":
self.input = ops.FileReader(file_root = "/data/imagenet/train-jpeg",
shard_id = 0,
num_shards = num_shards,
shard_id = 0,
num_shards = num_shards,
stick_to_shard = True,
skip_cached_images = skip_cached_images,
skip_cached_images = skip_cached_images,
prefetch_queue_depth = 1)

elif reader_type == "TFRecordReader":
tfrecord = sorted(glob.glob("/data/imagenet/train-val-tfrecord-480/train-*"))
tfrecord_idx = sorted(glob.glob("/data/imagenet/train-val-tfrecord-480.idx/train-*"))
self.input = ops.TFRecordReader(path = tfrecord,
index_path = tfrecord_idx,
shard_id = 0,
num_shards = num_shards,
shard_id = 0,
num_shards = num_shards,
stick_to_shard = True,
skip_cached_images = skip_cached_images,
skip_cached_images = skip_cached_images,
features = {"image/encoded" : tfrec.FixedLenFeature((), tfrec.string, ""),
"image/class/label": tfrec.FixedLenFeature([1], tfrec.int64, -1)})

Expand All @@ -1142,9 +1142,9 @@ def __init__(self, reader_type, batch_size, is_cached=False, seed=123456, skip_c
cache_threshold=0,
cache_type='threshold',
cache_debug=False,
cache_batch_copy=True)
cache_batch_copy=is_cached_batch_copy)
else:
self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)
self.decode = ops.nvJPEGDecoder(device = "mixed", output_type = types.RGB)

def define_graph(self):
if self.reader_type == "TFRecordReader":
Expand All @@ -1159,6 +1159,14 @@ def define_graph(self):
def iter_setup(self):
pass


def test_nvjpeg_cached_batch_copy_pipelines():
batch_size = 26
for reader_type in {"MXNetReader", "CaffeReader", "Caffe2Reader", "FileReader", "TFRecordReader"}:
compare_pipelines(CachedPipeline(reader_type, batch_size, is_cached=True, is_cached_batch_copy=True),
CachedPipeline(reader_type, batch_size, is_cached=True, is_cached_batch_copy=False),
batch_size=batch_size, N_iterations=20)

def test_nvjpeg_cached_pipelines():
batch_size = 26
for reader_type in {"MXNetReader", "CaffeReader", "Caffe2Reader", "FileReader", "TFRecordReader"}:
Expand Down