You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
warnings.warn(
proc_id: 0
world size: 1
local_rank: 0
Let's use 1 GPUs!
Loading Dataset ...
/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/numpy/core/fromnumeric.py:3474: RuntimeWarning: Mean of empty slice.
return _methods._mean(a, axis=axis, dtype=dtype,
/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/numpy/core/_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars
ret = ret.dtype.type(ret / rcount)
mean_cam_height nan, mean_cam_pitch nan
Traceback (most recent call last):
File "main_persformer.py", line 44, in
main()
File "main_persformer.py", line 35, in main
runner = Runner(args)
File "/mnt/data/HXX/PersFormer_3DLane/experiments/runner.py", line 68, in init
self.train_dataset, self.train_loader, self.train_sampler = self._get_train_dataset()
File "/mnt/data/HXX/PersFormer_3DLane/experiments/runner.py", line 685, in _get_train_dataset
train_dataset = LaneDataset(args.dataset_dir, args.data_dir + 'training/', args, data_aug=True, save_std=True, seg_bev=args.seg_bev)
File "/mnt/data/HXX/PersFormer_3DLane/data/Load_Data.py", line 272, in init
self._im_anchor_angles = self.init_dataset_openlane_beta(dataset_base_dir, json_file_path)
File "/mnt/data/HXX/PersFormer_3DLane/data/Load_Data.py", line 1588, in init_dataset_openlane_beta
lane_x_off_std = np.sqrt(np.average(lane_x_off_all**2, weights=visibility_all_flat, axis=0))
File "<array_function internals>", line 180, in average
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/numpy/lib/function_base.py", line 524, in average
raise ZeroDivisionError(
ZeroDivisionError: Weights sum to zero, can't be normalized
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 56497) of binary: /home/user/anaconda3/envs/Persformer_hxx/bin/python
Traceback (most recent call last):
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/torch/distributed/launch.py", line 193, in
main()
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/torch/distributed/launch.py", line 189, in main
launch(args)
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/torch/distributed/launch.py", line 174, in launch
run(args)
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/torch/distributed/run.py", line 710, in run
elastic_launch(
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 259, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
warnings.warn(
proc_id: 0
world size: 1
local_rank: 0
Let's use 1 GPUs!
Loading Dataset ...
/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/numpy/core/fromnumeric.py:3474: RuntimeWarning: Mean of empty slice.
return _methods._mean(a, axis=axis, dtype=dtype,
/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/numpy/core/_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars
ret = ret.dtype.type(ret / rcount)
mean_cam_height nan, mean_cam_pitch nan
Traceback (most recent call last):
File "main_persformer.py", line 44, in
main()
File "main_persformer.py", line 35, in main
runner = Runner(args)
File "/mnt/data/HXX/PersFormer_3DLane/experiments/runner.py", line 68, in init
self.train_dataset, self.train_loader, self.train_sampler = self._get_train_dataset()
File "/mnt/data/HXX/PersFormer_3DLane/experiments/runner.py", line 685, in _get_train_dataset
train_dataset = LaneDataset(args.dataset_dir, args.data_dir + 'training/', args, data_aug=True, save_std=True, seg_bev=args.seg_bev)
File "/mnt/data/HXX/PersFormer_3DLane/data/Load_Data.py", line 272, in init
self._im_anchor_angles = self.init_dataset_openlane_beta(dataset_base_dir, json_file_path)
File "/mnt/data/HXX/PersFormer_3DLane/data/Load_Data.py", line 1588, in init_dataset_openlane_beta
lane_x_off_std = np.sqrt(np.average(lane_x_off_all**2, weights=visibility_all_flat, axis=0))
File "<array_function internals>", line 180, in average
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/numpy/lib/function_base.py", line 524, in average
raise ZeroDivisionError(
ZeroDivisionError: Weights sum to zero, can't be normalized
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 56497) of binary: /home/user/anaconda3/envs/Persformer_hxx/bin/python
Traceback (most recent call last):
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/torch/distributed/launch.py", line 193, in
main()
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/torch/distributed/launch.py", line 189, in main
launch(args)
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/torch/distributed/launch.py", line 174, in launch
run(args)
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/torch/distributed/run.py", line 710, in run
elastic_launch(
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "/home/user/anaconda3/envs/Persformer_hxx/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 259, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
main_persformer.py FAILED
Failures:
<NO_OTHER_FAILURES>
Root Cause (first observed failure):
[0]:
time : 2023-04-17_17:21:54
host : ubuntu
rank : 0 (local_rank: 0)
exitcode : 1 (pid: 56497)
error_file: <N/A>
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
The text was updated successfully, but these errors were encountered: