Segmentation fault

[wangguangyuan]

python-2.7.5
tensorflow-gpu-1.8/1.7
cuda-9.0
cudnn-7.0.5
Titan V
my code:

`class CommonPipeline(Pipeline):
def init(self,
batch_size,
num_threads,
device_id,
size=(224, 224),
crop_size=(224, 224),
mean=IMAGE_MEAN,
std=IMAGE_STD,
channel_format=types.NHWC,
probability=0.5,
device='gpu',
decode_method='in_gpu'):

    super(CommonPipeline, self).__init__(batch_size, num_threads, device_id)

    if decode_method == 'in_gpu':
        self.decode = ops.nvJPEGDecoder(device="mixed", output_type=types.RGB)
    else:
        self.decode = ops.HostDecoder(output_type=types.RGB)

    self.rrc = ops.RandomResizedCrop(device=device, size=size)
    self.cmnp = ops.CropMirrorNormalize(device=device,
                                        output_dtype=types.FLOAT,
                                        output_layout=channel_format,
                                        image_type=types.RGB,
                                        crop=crop_size,
                                        mean=mean,
                                        std=std)
    self.coin = ops.CoinFlip(probability=probability)

def base_define_graph(self, inputs, labels):
    rng = self.coin()
    images = self.decode(inputs)
    images = self.rrc(images)
    images = self.cmnp(images, mirror=rng)
    return images, labels

class FileReadPipeline(CommonPipeline):
def init(self,
batch_size,
num_threads,
device_id,
file_root='',
file_list='',
size=(224, 224),
crop_size=(224, 224),
mean=IMAGE_MEAN,
std=IMAGE_STD,
channel_format=types.NHWC,
probability=0.5,
device='gpu',
decode_method='in_gpu'):
super(FileReadPipeline, self).init(batch_size=batch_size,
num_threads=num_threads,
device_id=device_id,
size=size,
crop_size=crop_size,
mean=mean,
std=std,
channel_format=channel_format,
probability=probability,
device=device,
decode_method=decode_method)
self.input = ops.FileReader(file_root=file_root, file_list=file_list, random_shuffle=True, initial_fill=21)

def define_graph(self):
    images, labels = self.input()
    return self.base_define_graph(images, labels)

class TFRecordPipeline(CommonPipeline):
pass

def get_batch_images(pipe_name,
batch_size,
num_threads,
device_id,
file_root='',
file_list='',
size=(224, 224),
crop_size=(224, 224),
mean=IMAGE_MEAN,
std=IMAGE_STD,
channel_format=types.NHWC,
probability=0.5,
device='gpu',
decode_method='in_gpu'):

pipes = [pipe_name(batch_size=batch_size,
                   num_threads=num_threads,
                   device_id=device_id,
                   file_root=file_root,
                   file_list=file_list,
                   size=size,
                   crop_size=crop_size,
                   mean=mean,
                   std=std,
                   channel_format=channel_format,
                   probability=probability,
                   device=device,
                   decode_method=decode_method)]

serialized_pipes = [pipe.serialize() for pipe in pipes]
del pipes
daliop_t = dali_tf.DALIIterator()

images = []
labels = []

print('/gpu:%i' % device_id)
with tf.device('/gpu:%i' % device_id):
    image, label = daliop_t(serialized_pipeline=serialized_pipes[0],
                            batch_size=batch_size,
                            height=crop_size[0],
                            width=crop_size[1],
                              # num_threads=num_threads,
                            device_id=device_id)
    images.append(image)
    labels.append(label)

    return images, labels

if name == 'main':
img_batch, label_batch = get_batch_images(FileReadPipeline, batch_size=1, num_threads=4,
device_id=0, file_root='images')
print(img_batch)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3)
config = tf.ConfigProto(gpu_options=gpu_options)
with tf.Session(config=config) as sess:
ims = sess.run(img_batch)
print(ims.shape)`

error:

[root]$ python dataloader.py
RuntimeError: module compiled against API version 0xc but this version of numpy is 0xb
RuntimeError: module compiled against API version 0xc but this version of numpy is 0xb
read 21 files from 2 directories
/gpu:0
[<tf.Tensor 'Dali:0' shape=(1, 224, 224, 3) dtype=float32>]
2018-07-26 18:49:03.560989: I tensorflow/core/platform/cpu_feature_guard.cc:140] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-07-26 18:49:03.564573: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1344] Found device 0 with properties:
name: TITAN Xp major: 6 minor: 1 memoryClockRate(GHz): 1.911
pciBusID: 0000:02:00.0
totalMemory: 11.90GiB freeMemory: 11.73GiB
2018-07-26 18:49:03.825189: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1344] Found device 1 with properties:
name: TITAN Xp major: 6 minor: 1 memoryClockRate(GHz): 1.911
pciBusID: 0000:83:00.0
totalMemory: 11.90GiB freeMemory: 11.74GiB
2018-07-26 18:49:03.825291: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1423] Adding visible gpu devices: 0, 1
2018-07-26 18:49:04.313040: I tensorflow/core/common_runtime/gpu/gpu_device.cc:911] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-07-26 18:49:04.313090: I tensorflow/core/common_runtime/gpu/gpu_device.cc:917] 0 1
2018-07-26 18:49:04.313100: I tensorflow/core/common_runtime/gpu/gpu_device.cc:930] 0: N N
2018-07-26 18:49:04.313107: I tensorflow/core/common_runtime/gpu/gpu_device.cc:930] 1: N N
2018-07-26 18:49:04.313616: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 3656 MB memory) -> physical GPU (device: 0, name: TITAN Xp, pci bus id: 0000:02:00.0, compute capability: 6.1)
2018-07-26 18:49:04.347755: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 3656 MB memory) -> physical GPU (device: 1, name: TITAN Xp, pci bus id: 0000:83:00.0, compute capability: 6.1)
read 21 files from 2 directories
Segmentation fault

[JanuszL]

Hi,
You forgot to add .gpu() to base_define_graph:

def base_define_graph(self, inputs, labels):
    rng = self.coin()
    images = self.decode(inputs)
    images = self.rrc(images)
    images = self.cmnp(images, mirror=rng)
    return images, labels.gpu()

TF expects that label and images are located in gpu memory. For images it is done by nvJPEGDecoder, for labels you need to offload them calling .gpu().

[wangguangyuan]

Thank you very much, I am negligent, indeed when it’s problem

[cliffwoolley]

Glad you got it working!