Using Colab error (if project created in Windows, Colab runs as Ubuntu- path fix needed)

[byronrogers73]

Your Operating system and DeepLabCut version

Using GoogleColab, DLC 2.0.3

Describe the problem

As per instructions copied the DLC folders created locally on my PC to Drive, no issues with any of the setup on the Colab notebook. Changed the config.yaml and pose_cfg.yaml folders to reflect the Drive locations, e.g 'snapshot_prefix': '/content/drive/MyDrive/DLC/ThoroughbredDLC-Byron-2018-12-08/dlc-models/iteration-0/ThoroughbredDLCDec8-trainset95shuffle1/train/snapshot',

Error generated....

Exception in thread Thread-5:
Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/usr/lib/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.6/dist-packages/deeplabcut/pose_estimation_tensorflow/train.py", line 51, in load_and_enqueue
batch_np = dataset.next_batch()
File "/usr/local/lib/python3.6/dist-packages/deeplabcut/pose_estimation_tensorflow/dataset/pose_dataset.py", line 188, in next_batch
return self.make_batch(data_item, scale, mirror)
File "/usr/local/lib/python3.6/dist-packages/deeplabcut/pose_estimation_tensorflow/dataset/pose_dataset.py", line 215, in make_batch
image = imread(os.path.join(self.cfg.project_path,im_file), mode='RGB')
File "/usr/local/lib/python3.6/dist-packages/numpy/lib/utils.py", line 101, in newfunc
return func(*args, **kwds)
File "/usr/local/lib/python3.6/dist-packages/scipy/misc/pilutil.py", line 164, in imread
im = Image.open(name)
File "/usr/local/lib/python3.6/dist-packages/PIL/Image.py", line 2312, in open
class ImageTransformHandler(object):
FileNotFoundError: [Errno 2] No such file or directory: '/content/drive/My Drive/DLC/ThoroughbredDLC-Byron-2018-12-08/labeled-data\Sample34\img077.png'

---

NotFoundError Traceback (most recent call last)

in ()
1
----> 2 deeplabcut.train_network(path_config_file, shuffle=1, displayiters=10,saveiters=500)
3
4 #this will run until you stop it (CTRL+C), or hit "STOP" icon, or when it hits the end (default, 1.03M iterations).
5 #Whichever you chose, you will see what looks like an error message, but it's not an error - don't worry....

/usr/local/lib/python3.6/dist-packages/deeplabcut/pose_estimation_tensorflow/training.py in train_network(config, shuffle, trainingsetindex, gputouse, max_snapshots_to_keep, autotune, displayiters, saveiters, maxiters)
79 train(str(poseconfigfile),displayiters,saveiters,maxiters,max_to_keep=max_snapshots_to_keep) #pass on path and file name for pose_cfg.yaml!
80 except BaseException as e:
---> 81 raise e
82 finally:
83 os.chdir(str(start_path))

[byronrogers73]

NotFoundError Traceback (most recent call last)

in ()
1
----> 2 deeplabcut.train_network(path_config_file, shuffle=1, displayiters=10,saveiters=500)
3
4 #this will run until you stop it (CTRL+C), or hit "STOP" icon, or when it hits the end (default, 1.03M iterations).
5 #Whichever you chose, you will see what looks like an error message, but it's not an error - don't worry....

/usr/local/lib/python3.6/dist-packages/deeplabcut/pose_estimation_tensorflow/training.py in train_network(config, shuffle, trainingsetindex, gputouse, max_snapshots_to_keep, autotune, displayiters, saveiters, maxiters)
79 train(str(poseconfigfile),displayiters,saveiters,maxiters,max_to_keep=max_snapshots_to_keep) #pass on path and file name for pose_cfg.yaml!
80 except BaseException as e:
---> 81 raise e
82 finally:
83 os.chdir(str(start_path))

/usr/local/lib/python3.6/dist-packages/deeplabcut/pose_estimation_tensorflow/training.py in train_network(config, shuffle, trainingsetindex, gputouse, max_snapshots_to_keep, autotune, displayiters, saveiters, maxiters)
77 os.environ['CUDA_VISIBLE_DEVICES'] = str(gputouse)
78 try:
---> 79 train(str(poseconfigfile),displayiters,saveiters,maxiters,max_to_keep=max_snapshots_to_keep) #pass on path and file name for pose_cfg.yaml!
80 except BaseException as e:
81 raise e

/usr/local/lib/python3.6/dist-packages/deeplabcut/pose_estimation_tensorflow/train.py in train(config_yaml, displayiters, saveiters, maxiters, max_to_keep)
107
108 # Restore variables from disk.
--> 109 restorer.restore(sess, cfg.init_weights)
110 if maxiters==None:
111 max_iter = int(cfg.multi_step[-1][1])

/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py in restore(self, sess, save_path)
1534 raise ValueError("Can't load save_path when it is None.")
1535
-> 1536 if not checkpoint_management.checkpoint_exists(compat.as_text(save_path)):
1537 raise ValueError("The passed save_path is not a valid checkpoint: "
1538 + compat.as_text(save_path))

/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/checkpoint_management.py in checkpoint_exists(checkpoint_prefix)
362 pathname = _prefix_to_checkpoint_path(checkpoint_prefix,
363 saver_pb2.SaverDef.V2)
--> 364 if file_io.get_matching_files(pathname):
365 return True
366 elif file_io.get_matching_files(checkpoint_prefix):

/usr/local/lib/python3.6/dist-packages/tensorflow/python/lib/io/file_io.py in get_matching_files(filename)
340 # Convert the filenames to string from bytes.
341 compat.as_str_any(matching_filename)
--> 342 for single_filename in filename
343 for matching_filename in pywrap_tensorflow.GetMatchingFiles(
344 compat.as_bytes(single_filename), status)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/errors_impl.py in exit(self, type_arg, value_arg, traceback_arg)
526 None, None,
527 compat.as_text(c_api.TF_Message(self.status.status)),
--> 528 c_api.TF_GetCode(self.status.status))
529 # Delete the underlying status object from memory otherwise it stays alive
530 # as there is a reference to status from this from the traceback due to

NotFoundError: /content/drive/My Drive/DLC/lib/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained; No such file or directory

[adam-matic]

We had the same error.
FileNotFoundError: [Errno 2] No such file or directory: '/content/drive/My Drive/DLC/ThoroughbredDLC-Byron-2018-12-08/labeled-data\Sample34\img077.png'

I suppose you are using a windows computer to label the video. Notice the mix of forward and backward slashes in the image path. Colab seems to be running on linux.

A temporary solution that worked for us was to edit the labeling_toolbox.py file. After line
227: self.index = glob.glob(os.path.join(self.dir,'*.png'))
add a line that replaces the windows-style backslashes to forward slashes:
self.index = [ s.replace("\\", "/" for s in self.index ]
I did not check if this still works in windows, so it probably isn't a general solution, but the generated project was good in colab

[MMathisLab]

Ah, simple fix to change the paths to Linux style, we have helper code for this; will put up on DLCUtils

[byronrogers73]

Thanks.

I was actually looking at the bottom error "NotFoundError: /content/drive/My Drive/DLC/lib/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained; No such file or directory" and missed the "/" error.

Let me know when it is up in DLCUtils.

[AlexEMG]

I put a quick fix on https://github.com/AlexEMG/DLCutils, see https://github.com/AlexEMG/DLCutils#running-project-created-on-windows-on-colaboratory

We will integrate this functionality in the next release of DLC (in an expanded way). Let me know if it works & makes sense!

[MargaridaMP]

Hi!

I believe I am still experiencing this issue.
I labelled the frames on a Windows PC (DLC version 2.1.6) and I am now on Colab (DLC version 2.1.7.1). During the 'create training dataset' step I get this:

FileNotFoundError: No such file: '/content/drive/My Drive/LP_Task_Nov19-Margarida-2020-04-03/labeled-data\LV_M8_S14\img29277.png'

Since the newer versions of DLC already fix for this issue, is there anything that I am doing wrong? How can I fix this?

Thank you!

[MMathisLab]

you can covert your files to linux style first, but it should convert when you run create_training_dataset; can you put the full code you ran and output here?

[MargaridaMP]

Code:

deeplabcut.create_training_dataset(path_config_file, net_type='resnet_50', augmenter_type='imgaug')

Ouput:

FileNotFoundError                         Traceback (most recent call last)
<ipython-input-81-d43c96cb30c9> in <module>()
----> 1 deeplabcut.create_training_dataset(path_config_file, net_type='resnet_50', augmenter_type='imgaug')

9 frames
/usr/local/lib/python3.6/dist-packages/deeplabcut/generate_training_dataset/trainingsetmanipulation.py in create_training_dataset(config, num_shuffles, Shuffles, windows2linux, userfeedback, trainIndexes, testIndexes, net_type, augmenter_type)
    690             # Saving data file (convert to training file for deeper cut (*.mat))
    691             ################################################################################
--> 692             data, MatlabData = format_training_data(Data, trainIndexes, nbodyparts, project_path)
    693             sio.savemat(os.path.join(project_path,datafilename), {'dataset': MatlabData})
    694 

/usr/local/lib/python3.6/dist-packages/deeplabcut/generate_training_dataset/trainingsetmanipulation.py in format_training_data(df, train_inds, nbodyparts, project_path)
    546         filename = df.index[i]
    547         data['image'] = filename
--> 548         img_shape = _read_image_shape_fast(os.path.join(project_path, filename))
    549         try:
    550             data['size'] = img_shape[2], img_shape[0], img_shape[1]

/usr/local/lib/python3.6/dist-packages/deeplabcut/generate_training_dataset/trainingsetmanipulation.py in _read_image_shape_fast(path)
    531 @lru_cache(maxsize=None)
    532 def _read_image_shape_fast(path):
--> 533     return io.imread(path).shape
    534 
    535 def format_training_data(df, train_inds, nbodyparts, project_path):

/usr/local/lib/python3.6/dist-packages/skimage/io/_io.py in imread(fname, as_gray, plugin, **plugin_args)
     46 
     47     with file_or_url_context(fname) as fname:
---> 48         img = call_plugin('imread', fname, plugin=plugin, **plugin_args)
     49 
     50     if not hasattr(img, 'ndim'):

/usr/local/lib/python3.6/dist-packages/skimage/io/manage_plugins.py in call_plugin(kind, *args, **kwargs)
    208                                (plugin, kind))
    209 
--> 210     return func(*args, **kwargs)
    211 
    212 

/usr/local/lib/python3.6/dist-packages/skimage/io/_plugins/imageio_plugin.py in imread(*args, **kwargs)
      8 @wraps(imageio_imread)
      9 def imread(*args, **kwargs):
---> 10     return np.asarray(imageio_imread(*args, **kwargs))

/usr/local/lib/python3.6/dist-packages/imageio/core/functions.py in imread(uri, format, **kwargs)
    219 
    220     # Get reader and read first
--> 221     reader = read(uri, format, "i", **kwargs)
    222     with reader:
    223         return reader.get_data(0)

/usr/local/lib/python3.6/dist-packages/imageio/core/functions.py in get_reader(uri, format, mode, **kwargs)
    128 
    129     # Create request object
--> 130     request = Request(uri, "r" + mode, **kwargs)
    131 
    132     # Get format

/usr/local/lib/python3.6/dist-packages/imageio/core/request.py in __init__(self, uri, mode, **kwargs)
    123 
    124         # Parse what was given
--> 125         self._parse_uri(uri)
    126 
    127         # Set extension

/usr/local/lib/python3.6/dist-packages/imageio/core/request.py in _parse_uri(self, uri)
    271                 # Reading: check that the file exists (but is allowed a dir)
    272                 if not os.path.exists(fn):
--> 273                     raise FileNotFoundError("No such file: '%s'" % fn)
    274             else:
    275                 # Writing: check that the directory to write to does exist

FileNotFoundError: No such file: '/content/drive/My Drive/LP_Task_Nov19-Margarida-2020-04-03/labeled-data\LV_M12_S14\img017712.png'

[MMathisLab]

can you also pass windows2linux=True when you run that call

[MMathisLab]

i.e.

deeplabcut.create_training_dataset(path_config_file, net_type='resnet_50', augmenter_type='imgaug', windows2linux=True)

[MargaridaMP]

It solved the issue!
Thanks a lot for your help.