how to read the image.list by pipeline

[liuguiyangnwpu]

hi,
I sorry to bother you.
I want to make my own dataset, but there is a problem that hold me the whole day. This is the code

import os
import sys
import tensorflow as tf
import cv2

readfile = "/Users/liuguiyang/Documents/CodeProj/PyProj/dtb/scripts/inputs/data/airplane/positive/test.list"
filenames = []
with open(readfile, 'r') as h:
    filenames = [f.strip() for f in h.readlines()]
# print(filenames)

def readMyFileFormatImg(fileNameQueue):
    reader = tf.TextLineReader(skip_header_lines=False)
    key, value = reader.read(fileNameQueue)
    raw_img = tf.read_file(key)
    # features = tf.image.decode_png(tf.read_file(value), channels=3)
    # features.set_shape((32,32,3))
    label = tf.stack([1])
    # label.set_shape((1,))
    return raw_img, label

def inputPipeLine(fileNames, batchSize = 4, numEpochs = None):
    fileNameQueue = tf.train.string_input_producer(fileNames, num_epochs = numEpochs)
    example, label = readMyFileFormatImg(fileNameQueue)
    # min_after_dequeue = 8
    # capacity = min_after_dequeue + 3 * batchSize
    # exampleBatch, labelBatch = tf.train.shuffle_batch([example, label], 
    #                                                    batch_size = batchSize, 
    #                                                    num_threads = 3,  
    #                                                    capacity = capacity, 
    #                                                    min_after_dequeue = min_after_dequeue)
    return example, label


featureBatch, labelBatch = inputPipeLine(filenames, batchSize = 4)
init = [
            tf.variables_initializer(tf.global_variables() +
                                     tf.local_variables()),
            tf.tables_initializer()
        ]
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
    sess.run(init)
    # Start populating the filename queue.                                                                                                                                                                                                                                    
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(coord=coord)

   # Retrieve a single instance:                                                                                                                                                                                                                                             
    try:
        #while not coord.should_stop():                                                                                                                                                                                                                                       
        while True:
            example, label = sess.run([featureBatch, labelBatch])
            print(example)
    except tf.errors.OutOfRangeError:
        print('Done reading')
    finally:
        coord.request_stop()

    coord.join(threads)
    sess.close()

The Error Info Msg is:

Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1022, in _do_call
    return fn(*args)
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1004, in _run_fn
    status, run_metadata)
  File "/usr/local/Cellar/python3/3.6.0/Frameworks/Python.framework/Versions/3.6/lib/python3.6/contextlib.py", line 89, in __exit__
    next(self.gen)
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py", line 469, in raise_exception_on_not_ok_status
    pywrap_tensorflow.TF_GetCode(status))
tensorflow.python.framework.errors_impl.NotFoundError: /Users/liuguiyang/Documents/CodeProj/PyProj/dtb/scripts/inputs/data/airplane/positive/32size/0335.png:1
	 [[Node: ReadFile = ReadFile[_device="/job:localhost/replica:0/task:0/cpu:0"](ReaderReadV2)]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "tune-airplane.py", line 60, in <module>
    example, label = sess.run([featureBatch, labelBatch])
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 767, in run
    run_metadata_ptr)
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 965, in _run
    feed_dict_string, options, run_metadata)
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1015, in _do_run
    target_list, options, run_metadata)
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1035, in _do_call
    raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.NotFoundError: /Users/liuguiyang/Documents/CodeProj/PyProj/dtb/scripts/inputs/data/airplane/positive/32size/0335.png:1
	 [[Node: ReadFile = ReadFile[_device="/job:localhost/replica:0/task:0/cpu:0"](ReaderReadV2)]]

Caused by op 'ReadFile', defined at:
  File "tune-airplane.py", line 44, in <module>
    featureBatch, labelBatch = inputPipeLine(filenames, batchSize = 4)
  File "tune-airplane.py", line 33, in inputPipeLine
    example, label = readMyFileFormatImg(fileNameQueue)
  File "tune-airplane.py", line 24, in readMyFileFormatImg
    raw_img = tf.read_file(key)
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gen_io_ops.py", line 202, in read_file
    result = _op_def_lib.apply_op("ReadFile", filename=filename, name=name)
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
    op_def=op_def)
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2395, in create_op
    original_op=self._default_original_op, op_def=op_def)
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1264, in __init__
    self._traceback = _extract_stack()

NotFoundError (see above for traceback): /Users/liuguiyang/Documents/CodeProj/PyProj/dtb/scripts/inputs/data/airplane/positive/32size/0335.png:1
	 [[Node: ReadFile = ReadFile[_device="/job:localhost/replica:0/task:0/cpu:0"](ReaderReadV2)]]

please give me some info.

guiyang, thx!

[galeone]

As you can see from the error

NotFoundError (see above for traceback): /Users/liuguiyang/Documents/CodeProj/PyProj/dtb/scripts/inputs/data/airplane/positive/32size/0335.png:1

The file /Users/liuguiyang/Documents/CodeProj/PyProj/dtb/scripts/inputs/data/airplane/positive/32size/0335.png:1 does not exist.

Check the content of image.list and look if the files listed there effectively exists.

(the :1 in the file name sounds strange to me, check this)

[liuguiyangnwpu]

hi,
after i check the imag.list, I can read the *.png images.

import os
import tensorflow as tf

filename = "/Users/liuguiyang/Documents/CodeProj/PyProj/dtb/scripts/inputs/data/airplane/positive/32size/0335.png"

def test02():
#    filename = "/Users/liuguiyang/Desktop/cifar/test/0_cat.png"
    with tf.Graph().as_default():
        image_contents = tf.read_file(filename)
        image = tf.image.decode_png(image_contents, channels=3)
        init_op = tf.tables_initializer()
        with tf.Session() as sess:
            sess.run(init_op)
            tmp = sess.run(image)
            print(tmp)
if __name__ == "__main__":
    test02()

the answer is below:

➜  dtb git:(CAE) ✗ python3 run.py > a.txt
➜  dtb git:(CAE) ✗ more a.txt 
[[[173 173 173]
  [173 173 173]
  [173 173 173]
  ..., 
  [176 176 176]
  [100 100 100]
  [117 117 117]]

 [[175 175 175]
  [174 174 174]
  [173 173 173]
  ..., 
  [177 177 177]
  [149 149 149]
  [136 136 136]]

 [[175 175 175]
  [173 173 173]
  [171 171 171]
  ..., 
  [173 173 173]
  [173 173 173]
  [161 161 161]]

 ..., 
 [[173 173 173]
  [173 173 173]
  [172 172 172]
  ..., 
  [174 174 174]
  [173 173 173]
  [173 173 173]]

 [[173 173 173]
  [173 173 173]
  [173 173 173]
  ..., 
  [152 152 152]
  [171 171 171]
  [170 170 170]]

 [[172 172 172]
  [172 172 172]
  [172 172 172]
  ..., 
  [146 146 146]
  [170 170 170]
  [173 173 173]]]

it really can read the image data.
I am very confused about this question ???

[galeone]

Your input pipeline was correct (the commented part, not the modified version you posted here with comments added and debug operations added), the only problem I see is in the image.list file content.

The image you read here does not have the :1 added at the end and the read operation worked well. Thus if every file listed in image.list exists everything should work fine

[liuguiyangnwpu]

hi,
i am very confused why my code is error?
But in another way, I get the same effect, below is my code.

import tensorflow as tf

filename_queue = tf.train.string_input_producer(
    tf.train.match_filenames_once("/Users/liuguiyang/Documents/CodeProj/PyProj/dtb/scripts/inputs/data/airplane/positive/32size/*.png")
)

image_reader = tf.WholeFileReader()
key, img_file = image_reader.read(filename_queue)
img = tf.image.decode_png(img_file)

with tf.Session() as sess:
    tf.global_variables_initializer().run()
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(coord=coord)

    img_tensor = sess.run([img])
    print(img_tensor)

    coord.request_stop()
    coord.join(threads=threads)

thank you for your help ?