InvalidArgumentError: Reduction axis 1 is empty in shape [0,0] #494
Comments
|
same problem! |
|
#170 presents the same problem |
|
Same problem here! |
|
Maybe it's too late for solution, however, I had this issue while I was playing with anchor box scales in Faster R-CNN config. Just note, that Faster R-CNN architecture is the base for Mask R-CNN. The issue appeared when I left only large scales for anchor boxes ("scales: 2" in config file) for first stage anchor generator. Adding scales with value 1 solved the issue. I'm not sure about this, but I suspect that issue appears when some masks or boxes are very large and applying large scales leads to some specific overlapping of anchor boxes on image or simply the size of the anchor box gets larger than the image width/height. |
|
@KuznetsovIllya What do you mean by scales 1? The only thing I see in config file is RPN_ANCHOR_SCALES=(32, 64, 128, 256, 512). Do you change this? to what values? |
I'm using config files for training Faster R-CNN: |
Hello!
I am training the network on my own dataset, and I am running on an issue:
InvalidArgumentError: Reduction axis 1 is empty in shape [0,0]
I do not think my dataset is in cause, as it worked before (When I had 4096x2048 images, now I have 1024x1024)
Mightit be an issue with Keras or this implementation of the Mask R-CNN?
It seems it has something to do with ArgMax.
Below is the full error log.
Thank you!
InvalidArgumentError Traceback (most recent call last)
~\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
1326 try:
-> 1327 return fn(*args)
1328 except errors.OpError as e:
~\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
1311 return self._call_tf_sessionrun(
-> 1312 options, feed_dict, fetch_list, target_list, run_metadata)
1313
~\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
1419 self._session, options, feed_dict, fetch_list, target_list,
-> 1420 status, run_metadata)
1421
~\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\framework\errors_impl.py in exit(self, type_arg, value_arg, traceback_arg)
515 compat.as_text(c_api.TF_Message(self.status.status)),
--> 516 c_api.TF_GetCode(self.status.status))
517 # Delete the underlying status object from memory otherwise it stays alive
InvalidArgumentError: Reduction axis 1 is empty in shape [0,0]
[[Node: proposal_targets_1/ArgMax = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT64, _device="/job:localhost/replica:0/task:0/device:GPU:0"](proposal_targets_1/Gather_5, rpn_class_loss_1/Equal/y)]]
[[Node: mrcnn_class_loss_1/SparseSoftmaxCrossEntropyWithLogits/Shape/_11051 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_4781_mrcnn_class_loss_1/SparseSoftmaxCrossEntropyWithLogits/Shape", tensor_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]
During handling of the above exception, another exception occurred:
InvalidArgumentError Traceback (most recent call last)
in ()
6 learning_rate=config.LEARNING_RATE,
7 epochs=5,
----> 8 layers='heads')
~\TensorFlow\Master_RCNN_1204\mrcnn\model.py in train(self, train_dataset, val_dataset, learning_rate, epochs, layers, augmentation)
2315 max_queue_size=100,
2316 workers=workers,
-> 2317 use_multiprocessing=True,
2318 )
2319 self.epoch = max(self.epoch, epochs)
~\Anaconda3\envs\tensorflow\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your
' + object_name + 90 'call to the Keras 2 API: ' + signature, stacklevel=2)---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
~\Anaconda3\envs\tensorflow\lib\site-packages\keras\engine\training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
2222 outs = self.train_on_batch(x, y,
2223 sample_weight=sample_weight,
-> 2224 class_weight=class_weight)
2225
2226 if not isinstance(outs, list):
~\Anaconda3\envs\tensorflow\lib\site-packages\keras\engine\training.py in train_on_batch(self, x, y, sample_weight, class_weight)
1881 ins = x + y + sample_weights
1882 self._make_train_function()
-> 1883 outputs = self.train_function(ins)
1884 if len(outputs) == 1:
1885 return outputs[0]
~\Anaconda3\envs\tensorflow\lib\site-packages\keras\backend\tensorflow_backend.py in call(self, inputs)
2476 session = get_session()
2477 updated = session.run(fetches=fetches, feed_dict=feed_dict,
-> 2478 **self.session_kwargs)
2479 return updated[:len(self.outputs)]
2480
~\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in run(self, fetches, feed_dict, options, run_metadata)
903 try:
904 result = self._run(None, fetches, feed_dict, options_ptr,
--> 905 run_metadata_ptr)
906 if run_metadata:
907 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
~\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
1138 if final_fetches or final_targets or (handle and feed_dict_tensor):
1139 results = self._do_run(handle, final_targets, final_fetches,
-> 1140 feed_dict_tensor, options, run_metadata)
1141 else:
1142 results = []
~\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1319 if handle is None:
1320 return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1321 run_metadata)
1322 else:
1323 return self._do_call(_prun_fn, handle, feeds, fetches)
~\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
1338 except KeyError:
1339 pass
-> 1340 raise type(e)(node_def, op, message)
1341
1342 def _extend_graph(self):
InvalidArgumentError: Reduction axis 1 is empty in shape [0,0]
[[Node: proposal_targets_1/ArgMax = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT64, _device="/job:localhost/replica:0/task:0/device:GPU:0"](proposal_targets_1/Gather_5, rpn_class_loss_1/Equal/y)]]
[[Node: mrcnn_class_loss_1/SparseSoftmaxCrossEntropyWithLogits/Shape/_11051 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_4781_mrcnn_class_loss_1/SparseSoftmaxCrossEntropyWithLogits/Shape", tensor_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]
Caused by op 'proposal_targets_1/ArgMax', defined at:
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel_launcher.py", line 16, in
app.launch_new_instance()
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\traitlets\config\application.py", line 658, in launch_instance
app.start()
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\kernelapp.py", line 486, in start
self.io_loop.start()
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\tornado\platform\asyncio.py", line 112, in start
self.asyncio_loop.run_forever()
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\asyncio\base_events.py", line 421, in run_forever
self._run_once()
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\asyncio\base_events.py", line 1425, in _run_once
handle._run()
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\asyncio\events.py", line 127, in _run
self._callback(*self._args)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\tornado\platform\asyncio.py", line 102, in _handle_events
handler_func(fileobj, events)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\tornado\stack_context.py", line 276, in null_wrapper
return fn(*args, **kwargs)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\zmq\eventloop\zmqstream.py", line 450, in _handle_events
self._handle_recv()
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\zmq\eventloop\zmqstream.py", line 480, in _handle_recv
self._run_callback(callback, msg)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\zmq\eventloop\zmqstream.py", line 432, in _run_callback
callback(*args, **kwargs)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\tornado\stack_context.py", line 276, in null_wrapper
return fn(*args, **kwargs)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\kernelbase.py", line 233, in dispatch_shell
handler(stream, idents, msg)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\ipkernel.py", line 208, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\zmqshell.py", line 537, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\IPython\core\interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\IPython\core\interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\IPython\core\interactiveshell.py", line 2903, in run_ast_nodes
if self.run_code(code, result):
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\IPython\core\interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "", line 3, in
model_dir=MODEL_DIR)
File "C:\Users\edfla\TensorFlow\Master_RCNN_1204\mrcnn\model.py", line 1820, in init
self.keras_model = self.build(mode=mode, config=config)
File "C:\Users\edfla\TensorFlow\Master_RCNN_1204\mrcnn\model.py", line 1969, in build
target_rois, input_gt_class_ids, gt_boxes, input_gt_masks])
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\keras\engine\topology.py", line 619, in call
output = self.call(inputs, **kwargs)
File "C:\Users\edfla\TensorFlow\Master_RCNN_1204\mrcnn\model.py", line 660, in call
self.config.IMAGES_PER_GPU, names=names)
File "C:\Users\edfla\TensorFlow\Master_RCNN_1204\mrcnn\utils.py", line 825, in batch_slice
output_slice = graph_fn(*inputs_slice)
File "C:\Users\edfla\TensorFlow\Master_RCNN_1204\mrcnn\model.py", line 659, in
w, x, y, z, self.config),
File "C:\Users\edfla\TensorFlow\Master_RCNN_1204\mrcnn\model.py", line 563, in detection_targets_graph
roi_gt_box_assignment = tf.argmax(positive_overlaps, axis=1)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\util\deprecation.py", line 432, in new_func
return func(*args, **kwargs)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\ops\math_ops.py", line 214, in argmax
return gen_math_ops.arg_max(input, axis, name=name, output_type=output_type)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 818, in arg_max
name=name)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\framework\ops.py", line 3290, in create_op
op_def=op_def)
File "C:\Users\edfla\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\framework\ops.py", line 1654, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): Reduction axis 1 is empty in shape [0,0]
[[Node: proposal_targets_1/ArgMax = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT64, _device="/job:localhost/replica:0/task:0/device:GPU:0"](proposal_targets_1/Gather_5, rpn_class_loss_1/Equal/y)]]
[[Node: mrcnn_class_loss_1/SparseSoftmaxCrossEntropyWithLogits/Shape/_11051 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_4781_mrcnn_class_loss_1/SparseSoftmaxCrossEntropyWithLogits/Shape", tensor_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]
The text was updated successfully, but these errors were encountered: