cat sc_4294967294.err 100%|██████████████████████████████████████████|[00:06<00:00, 3.32s/it] 100%|██████████████████████████████████████████|[01:03<00:00, 2.53s/it] 100%|██████████████████████████████████████████|[03:38<00:00, 8.75s/it] 100%|██████████████████████████████████████████|[00:32<00:00, 16.10s/it] 100%|██████████████████████████████████████████|[00:28<00:00, 14.22s/it] 100%|██████████████████████████████████████████|[00:17<00:00, 4.48s/it] 100%|██████████████████████████████████████████|[00:00<00:00, 11.15it/s] 100%|██████████████████████████████████████████|[00:00<00:00, 11.40it/s] 100%|████████████████████████████████████████|[00:00<00:00, 1743.99it/s] 100%|██████████████████████████████████████████|[00:00<00:00, 10.50it/s] Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff36fd8970, win=0x7fa52ec75ad8) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff57bb7a60, win=0x7f2d8eec6a68) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff79637200, win=0x7fc7c767fad8) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff27c5df10, win=0x7fc9fc3c0a68) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fffbf6f33c0, win=0x7f7c72200a68) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff0d63fbe0, win=0x7fcb62571a30) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff669c30c0, win=0x7faec0eb6b10) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff8566c020, win=0x7efe75268aa0) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fffced1d5a0, win=0x7f2141f10aa0) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff49c2ae40, win=0x7f02aa96ca68) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff50538560, win=0x7ff66763ead8) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fffb356dcd0, win=0x7efe81e9aaa0) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fffac7a4130, win=0x7f5c4d83fa68) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff8a1c2710, win=0x7f6282458aa0) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff0703dac0, win=0x7f9baccf2aa0) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff7bf5a570, win=0x7f92c8313b48) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff0eb34ee0, win=0x7f5ab8b32a68) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7ffff60219e0, win=0x7fb00257eb80) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=2904, MPI_INFO_NULL, comm=0x84000004, baseptr=0x7fff7500f9e0, win=0x7fb5a172af70) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fffb0e5f940, win=0x7f09ea34b9f8) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7ffff6cc5150, win=0x7f3b0c25b9f8) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fffd6110eb0, win=0x7f283079fb10) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff2ae61050, win=0x7f0583db2b10) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus-subtask", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus-subtask')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/subtask.py", line 43, in main circus.launch(task, filename, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/__init__.py", line 21, in launch module.main(params, nb_cpu, nb_gpu, use_gpu) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/fitting.py", line 134, in main c_overs = io.load_data_memshared(params, 'overlaps') File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/shared/files.py", line 501, in load_data_memshared win_indices = MPI.Win.Allocate_shared(indices_bytes, intsize, comm=sub_comm) File "mpi4py/MPI/Win.pyx", line 101, in mpi4py.MPI.Win.Allocate_shared mpi4py.MPI.Exception: Other MPI error, error stack: PMPI_Win_allocate_shared(162).......: MPI_Win_allocate_shared(size=0, MPI_INFO_NULL, comm=0x84000002, baseptr=0x7fff9a10b810, win=0x7f0b8bc45ad8) failed MPID_Win_allocate_shared(208).......: win_init(276).......................: MPIR_Comm_dup_impl(57)..............: MPIR_Comm_copy(739).................: MPIR_Get_contextid_sparse_group(614): Too many communicators (0/2048 free on this process; ignore_id=0) Traceback (most recent call last): File "/home/s.sridhar/miniconda2/envs/spiking/bin/spyking-circus", line 11, in load_entry_point('spyking-circus==0.6.7', 'console_scripts', 'spyking-circus')() File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/site-packages/circus/scripts/launch.py", line 364, in main subprocess.check_call(mpi_args) File "/home/s.sridhar/miniconda2/envs/spiking/lib/python3.6/subprocess.py", line 291, in check_call raise CalledProcessError(retcode, cmd) subprocess.CalledProcessError: Command '['mpiexec', '-np', '24', 'spyking-circus-subtask', 'fitting', '/tmp/i140703-001.dat', '24', '0', 'False']' returned non-zero exit status 1.