Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ingest of scanpy cannot work because of problems in numpy in colab platform #19614

Closed
HelloWorldLTY opened this issue Aug 5, 2021 · 2 comments

Comments

@HelloWorldLTY
Copy link

Sorry to disturb, but it seems that the ingest method in scanpy meets some problems caused by pynndescent and numba. Here are the details:
KeyError Traceback (most recent call last)
/usr/local/lib/python3.7/dist-packages/numba/core/caching.py in save(self, key, data)
486 # If key already exists, we will overwrite the file
--> 487 data_name = overloads[key]
488 except KeyError:

KeyError: ((array(int32, 1d, C), array(int32, 1d, C), array(float32, 1d, C), array(float32, 2d, C), type(CPUDispatcher(<function squared_euclidean at 0x7fdefdb13830>)), array(int64, 1d, C), float64), ('x86_64-unknown-linux-gnu', 'broadwell', '+64bit,+adx,+aes,+avx,+avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vpopcntdq,+bmi,+bmi2,-cldemote,-clflushopt,-clwb,-clzero,+cmov,+cx16,+cx8,-enqcmd,+f16c,+fma,-fma4,+fsgsbase,+fxsr,-gfni,+invpcid,-lwp,+lzcnt,+mmx,+movbe,-movdir64b,-movdiri,-mwaitx,+pclmul,-pconfig,-pku,+popcnt,-prefetchwt1,+prfchw,-ptwrite,-rdpid,+rdrnd,+rdseed,+rtm,+sahf,-sgx,-sha,-shstk,+sse,+sse2,+sse3,+sse4.1,+sse4.2,-sse4a,+ssse3,-tbm,-vaes,-vpclmulqdq,-waitpkg,-wbnoinvd,-xop,+xsave,-xsavec,+xsaveopt,-xsaves'), ('308c49885ad3c35a475c360e21af1359caa88c78eb495fa0f5e8c6676ae5019e', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'))

During handling of the above exception, another exception occurred:

TypeError Traceback (most recent call last)
13 frames
in ()
----> 1 sc.tl.ingest(adata, adata_ref, obs='louvain') #ingest

/usr/local/lib/python3.7/dist-packages/scanpy/tools/_ingest.py in ingest(adata, adata_ref, obs, embedding_method, labeling_method, neighbors_key, inplace, **kwargs)
131
132 if obs is not None:
--> 133 ing.neighbors(**kwargs)
134 for i, col in enumerate(obs):
135 ing.map_labels(col, labeling_method[i])

/usr/local/lib/python3.7/dist-packages/scanpy/tools/_ingest.py in neighbors(self, k, queue_size, epsilon, random_state)
469 self._nnd_idx.search_rng_state = rng_state
470
--> 471 self._indices, self._distances = self._nnd_idx.query(test, k, epsilon)
472
473 else:

/usr/local/lib/python3.7/dist-packages/pynndescent/pynndescent_.py in query(self, query_data, k, epsilon)
1564 """
1565 if not hasattr(self, "_search_graph"):
-> 1566 self._init_search_graph()
1567
1568 if not self._is_sparse:

/usr/local/lib/python3.7/dist-packages/pynndescent/pynndescent_.py in _init_search_graph(self)
1061 self._distance_func,
1062 self.rng_state,
-> 1063 self.diversify_prob,
1064 )
1065 reverse_graph.eliminate_zeros()

/usr/local/lib/python3.7/dist-packages/numba/core/dispatcher.py in _compile_for_args(self, *args, **kws)
432 e.patch_message('\n'.join((str(e).rstrip(), help_msg)))
433 # ignore the FULL_TRACEBACKS config, this needs reporting!
--> 434 raise e
435
436 def inspect_llvm(self, signature=None):

/usr/local/lib/python3.7/dist-packages/numba/core/dispatcher.py in _compile_for_args(self, *args, **kws)
365 argtypes.append(self.typeof_pyval(a))
366 try:
--> 367 return self.compile(tuple(argtypes))
368 except errors.ForceLiteralArg as e:
369 # Received request for compiler re-entry with the list of arguments

/usr/local/lib/python3.7/dist-packages/numba/core/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
30 def _acquire_compile_lock(*args, **kwargs):
31 with self:
---> 32 return func(*args, **kwargs)
33 return _acquire_compile_lock
34

/usr/local/lib/python3.7/dist-packages/numba/core/dispatcher.py in compile(self, sig)
823 raise e.bind_fold_arguments(folded)
824 self.add_overload(cres)
--> 825 self._cache.save_overload(sig, cres)
826 return cres.entry_point
827

/usr/local/lib/python3.7/dist-packages/numba/core/caching.py in save_overload(self, sig, data)
669 """
670 with self._guard_against_spurious_io_errors():
--> 671 self._save_overload(sig, data)
672
673 def _save_overload(self, sig, data):

/usr/local/lib/python3.7/dist-packages/numba/core/caching.py in _save_overload(self, sig, data)
679 key = self._index_key(sig, _get_codegen(data))
680 data = self._impl.reduce(data)
--> 681 self._cache_file.save(key, data)
682
683 @contextlib.contextmanager

/usr/local/lib/python3.7/dist-packages/numba/core/caching.py in save(self, key, data)
494 break
495 overloads[key] = data_name
--> 496 self._save_index(overloads)
497 self._save_data(data_name, data)
498

/usr/local/lib/python3.7/dist-packages/numba/core/caching.py in _save_index(self, overloads)
540 def _save_index(self, overloads):
541 data = self._source_stamp, overloads
--> 542 data = self._dump(data)
543 with self._open_for_write(self._index_path) as f:
544 pickle.dump(self._version, f, protocol=-1)

/usr/local/lib/python3.7/dist-packages/numba/core/caching.py in _dump(self, obj)
568
569 def _dump(self, obj):
--> 570 return pickle.dumps(obj, protocol=-1)
571
572 @contextlib.contextmanager

TypeError: can't pickle weakref objects

How to solve this problem? Thanks.
The code comes from :https://scanpy-tutorials.readthedocs.io/en/latest/integrating-data-using-ingest.html

@HelloWorldLTY
Copy link
Author

For more details, please take a look at:
lmcinnes/pynndescent#133

@rkern
Copy link
Member

rkern commented Aug 5, 2021

It doesn't look like this has anything to do with numpy. Did you mean to post this to the numba repo?

@rkern rkern closed this as completed Aug 5, 2021
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

2 participants