module 'thinc_gpu_ops' has no attribute 'hash' #79

j2l · 2018-10-25T13:23:34Z

On Win10, thinc 6.12.0, spacy 2.0.16, cupy 4.1.0

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-5-2432a7701a48> in <module>()
      1 # defining doc
----> 2 doc = nlp("Jill laughed at John Johnson.")
      3 spacy.displacy.render(doc, style='dep', options={'distance' : 140}, jupyter=True)

E:\Anaconda3python\lib\site-packages\spacy\language.py in __call__(self, text, disable)
    344             if not hasattr(proc, '__call__'):
    345                 raise ValueError(Errors.E003.format(component=type(proc), name=name))
--> 346             doc = proc(doc)
    347             if doc is None:
    348                 raise ValueError(Errors.E005.format(name=name))

pipeline.pyx in spacy.pipeline.Tagger.__call__()

pipeline.pyx in spacy.pipeline.Tagger.predict()

E:\Anaconda3python\lib\site-packages\thinc\neural\_classes\model.py in __call__(self, x)
    159             Must match expected shape
    160         '''
--> 161         return self.predict(x)
    162 
    163     def pipe(self, stream, batch_size=128):

E:\Anaconda3python\lib\site-packages\thinc\api.py in predict(self, X)
     53     def predict(self, X):
     54         for layer in self._layers:
---> 55             X = layer(X)
     56         return X
     57 

E:\Anaconda3python\lib\site-packages\thinc\neural\_classes\model.py in __call__(self, x)
    159             Must match expected shape
    160         '''
--> 161         return self.predict(x)
    162 
    163     def pipe(self, stream, batch_size=128):

E:\Anaconda3python\lib\site-packages\thinc\api.py in predict(seqs_in)
    291     def predict(seqs_in):
    292         lengths = layer.ops.asarray([len(seq) for seq in seqs_in])
--> 293         X = layer(layer.ops.flatten(seqs_in, pad=pad))
    294         return layer.ops.unflatten(X, lengths, pad=pad)
    295 

E:\Anaconda3python\lib\site-packages\thinc\neural\_classes\model.py in __call__(self, x)
    159             Must match expected shape
    160         '''
--> 161         return self.predict(x)
    162 
    163     def pipe(self, stream, batch_size=128):

E:\Anaconda3python\lib\site-packages\thinc\api.py in predict(self, X)
     53     def predict(self, X):
     54         for layer in self._layers:
---> 55             X = layer(X)
     56         return X
     57 

E:\Anaconda3python\lib\site-packages\thinc\neural\_classes\model.py in __call__(self, x)
    159             Must match expected shape
    160         '''
--> 161         return self.predict(x)
    162 
    163     def pipe(self, stream, batch_size=128):

E:\Anaconda3python\lib\site-packages\thinc\neural\_classes\model.py in predict(self, X)
    123 
    124     def predict(self, X):
--> 125         y, _ = self.begin_update(X)
    126         return y
    127 

E:\Anaconda3python\lib\site-packages\thinc\api.py in uniqued_fwd(X, drop)
    372                                                     return_counts=True)
    373         X_uniq = layer.ops.xp.ascontiguousarray(X[ind])
--> 374         Y_uniq, bp_Y_uniq = layer.begin_update(X_uniq, drop=drop)
    375         Y = Y_uniq[inv].reshape((X.shape[0],) + Y_uniq.shape[1:])
    376         def uniqued_bwd(dY, sgd=None):

E:\Anaconda3python\lib\site-packages\thinc\api.py in begin_update(self, X, drop)
     59         callbacks = []
     60         for layer in self._layers:
---> 61             X, inc_layer_grad = layer.begin_update(X, drop=drop)
     62             callbacks.append(inc_layer_grad)
     63         def continue_update(gradient, sgd=None):

E:\Anaconda3python\lib\site-packages\thinc\api.py in begin_update(X, *a, **k)
    174     def begin_update(X, *a, **k):
    175         forward, backward = split_backward(layers)
--> 176         values = [fwd(X, *a, **k) for fwd in forward]
    177 
    178         output = ops.xp.hstack(values)

E:\Anaconda3python\lib\site-packages\thinc\api.py in <listcomp>(.0)
    174     def begin_update(X, *a, **k):
    175         forward, backward = split_backward(layers)
--> 176         values = [fwd(X, *a, **k) for fwd in forward]
    177 
    178         output = ops.xp.hstack(values)

E:\Anaconda3python\lib\site-packages\thinc\api.py in wrap(*args, **kwargs)
    256     '''
    257     def wrap(*args, **kwargs):
--> 258         output = func(*args, **kwargs)
    259         if splitter is None:
    260             to_keep, to_sink = output

E:\Anaconda3python\lib\site-packages\thinc\api.py in begin_update(X, *a, **k)
    174     def begin_update(X, *a, **k):
    175         forward, backward = split_backward(layers)
--> 176         values = [fwd(X, *a, **k) for fwd in forward]
    177 
    178         output = ops.xp.hstack(values)

E:\Anaconda3python\lib\site-packages\thinc\api.py in <listcomp>(.0)
    174     def begin_update(X, *a, **k):
    175         forward, backward = split_backward(layers)
--> 176         values = [fwd(X, *a, **k) for fwd in forward]
    177 
    178         output = ops.xp.hstack(values)

E:\Anaconda3python\lib\site-packages\thinc\api.py in wrap(*args, **kwargs)
    256     '''
    257     def wrap(*args, **kwargs):
--> 258         output = func(*args, **kwargs)
    259         if splitter is None:
    260             to_keep, to_sink = output

E:\Anaconda3python\lib\site-packages\thinc\api.py in begin_update(X, *a, **k)
    174     def begin_update(X, *a, **k):
    175         forward, backward = split_backward(layers)
--> 176         values = [fwd(X, *a, **k) for fwd in forward]
    177 
    178         output = ops.xp.hstack(values)

E:\Anaconda3python\lib\site-packages\thinc\api.py in <listcomp>(.0)
    174     def begin_update(X, *a, **k):
    175         forward, backward = split_backward(layers)
--> 176         values = [fwd(X, *a, **k) for fwd in forward]
    177 
    178         output = ops.xp.hstack(values)

E:\Anaconda3python\lib\site-packages\thinc\api.py in wrap(*args, **kwargs)
    256     '''
    257     def wrap(*args, **kwargs):
--> 258         output = func(*args, **kwargs)
    259         if splitter is None:
    260             to_keep, to_sink = output

E:\Anaconda3python\lib\site-packages\thinc\api.py in begin_update(X, *a, **k)
    174     def begin_update(X, *a, **k):
    175         forward, backward = split_backward(layers)
--> 176         values = [fwd(X, *a, **k) for fwd in forward]
    177 
    178         output = ops.xp.hstack(values)

E:\Anaconda3python\lib\site-packages\thinc\api.py in <listcomp>(.0)
    174     def begin_update(X, *a, **k):
    175         forward, backward = split_backward(layers)
--> 176         values = [fwd(X, *a, **k) for fwd in forward]
    177 
    178         output = ops.xp.hstack(values)

E:\Anaconda3python\lib\site-packages\thinc\api.py in wrap(*args, **kwargs)
    256     '''
    257     def wrap(*args, **kwargs):
--> 258         output = func(*args, **kwargs)
    259         if splitter is None:
    260             to_keep, to_sink = output

E:\Anaconda3python\lib\site-packages\thinc\neural\_classes\hash_embed.py in begin_update(self, ids, drop)
     49         if ids.ndim >= 2:
     50             ids = self.ops.xp.ascontiguousarray(ids[:, self.column], dtype='uint64')
---> 51         keys = self.ops.hash(ids, self.seed) % self.nV
     52         vectors = self.vectors[keys].sum(axis=1)
     53         mask = self.ops.get_dropout_mask((vectors.shape[1],), drop)

ops.pyx in thinc.neural.ops.CupyOps.hash()

AttributeError: module 'thinc_gpu_ops' has no attribute 'hash'

The text was updated successfully, but these errors were encountered:

pavanAlluri · 2018-10-29T18:12:45Z

I too am facing the same exact issue on same environment.
@ines , @honnibal Appreciate any pointers on resolving this?

Thank you for your time

folterj · 2018-11-01T21:13:28Z

I've got the same error (I'm on Ubuntu is I was unable to get spaCy+GPU to work on Windows 10)

...
File "/home/y/.local/lib/python3.6/site-packages/thinc/neural/_classes/hash_embed.py", line 51, in begin_update
keys = self.ops.hash(ids, self.seed) % self.nV
File "ops.pyx", line 821, in thinc.neural.ops.CupyOps.hash
AttributeError: module 'thinc_gpu_ops' has no attribute 'hash'

OS: Ubuntu 18.04
Python: 3.6.6
CUDA: 9.2
Spacy 2.0.16 (with cuda 9.2)
cupy-cuda92 5.0.0
cymem 2.0.2
thinc 6.12.0
thinc-gpu-ops 0.0.3

ZackAttack614 · 2018-11-03T01:34:18Z

Exactly the same issue here. Anxious for a resolution.

honnibal · 2018-11-04T14:29:00Z

I'm really not sure why it hasn't built correctly. If you look at the thinc_gpu_ops module, you can check whether the ops shared object compiled. Do you have a working compiler installed, specifically build-essential and python-dev?

I'm pretty confident this is an installation error, it's not a bug in the actual code. If you have a look here: https://github.com/explosion/thinc_gpu_ops/blob/master/thinc_gpu_ops/__init__.py you can see that we import * from the ops module, and then if you have a look in that module you'll see that indeed there's a function hash. So in the code everything is where it should be --- I think it's just not compiling correctly on your machines, somehow. Perhaps your cuda directory is in a non-default place, and cupy can't find the nvrc compiler?

folterj · 2018-11-12T14:51:12Z

Hi, I'm still unable to get spaCy working with GPU - just on Linux, failing on the above error (sorry for my previous cross-thread posting).

I'm about to get some serious CUDA cards, and am wondering if training&running spaCy NLP will work & benefit with this. Would you be able to recommend the (latest) versions which work with the current version of spaCy, using Python 3 on Linux? Any detail would be useful, even which Python IDE. It seems various including me are really keen on finding a working (software) configuration. Thanks again.

honnibal · 2018-12-02T04:54:21Z

Does cupy work for you? I'm really not sure what the problem might be. On my servers, with CUDA 9.2 installed pip install spacy[cuda92] works without trouble.

honnibal · 2018-12-02T04:56:18Z

Here's the script I use to install CUDA and CuDNN

#!/usr/bin/env bash

set -e

# Install driver
sudo add-apt-repository ppa:graphics-drivers/ppa
sudo apt-get update

sudo apt-get install -y nvidia-driver-396 libnvidia-compute-396 libnvidia-common-396 nvidia-utils-396

# Download toolkit and patch
wget -O cuda_9.2.88_396.26_linux.run -c https://developer.nvidia.com/compute/cuda/9.2/Prod/local_installers/cuda_9.2.88_396.26_linux --quiet
sudo mv cuda_9.2.88_396.26_linux.run /etc/install-cuda-9.2
wget https://developer.nvidia.com/compute/cuda/9.2/Prod/patches/1/cuda_9.2.88.1_linux --quiet
sudo mv cuda_9.2.88.1_linux /etc/patch-cuda-9.2
sudo chmod a+rx /etc/install-cuda-9.2
sudo chmod a+rx /etc/patch-cuda-9.2

sudo /etc/install-cuda-9.2 --toolkit --silent --verbose
sudo /etc/patch-cuda-9.2 --silent --accept-eula

sudo cp /tmp/binaries/cudnn-9.2-linux-x64-v7.1.tgz /etc/cudnn.tgz
sudo cp /tmp/runtime/cuda_bashrc /home/ubuntu/.bashrc
sudo chmod a+rwx /home/ubuntu/.bashrc

cd /tmp/binaries
tar -xzf cudnn-9.2-linux-x64-v7.1.tgz
sudo cp -r cuda/include/* /usr/local/cuda/include
sudo cp -r cuda/lib64/* /usr/local/cuda/lib64
sudo chmod -R a+rx /usr/local/cuda/include/*
sudo chmod -R a+rx /usr/local/cuda/lib64/*

honnibal · 2018-12-02T04:58:24Z

Did you set your LD_LIBRARY_PATH and PATH variables?

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64
export PATH=$PATH:/usr/local/cuda/bin

0x7r1574n · 2018-12-15T00:12:21Z

On macOS I've solved this error by first creating a symlink in the CUDA path (since the setup.py is looking for lib64 folder specifically and it is not present on macOS CUDA installations):

sudo ln -s /Developer/NVIDIA/CUDA-10.0/lib /Developer/NVIDIA/CUDA-10.0/lib64

and then run install with CUDA_HOME specified:

CUDA_HOME=/Developer/NVIDIA/CUDA-10.0 pip install thinc_gpu_ops

@honnibal I am just documenting my process and hopefully it will provide some pointers for you to fix this problem. Maybe there is something similar happening on OP's Windows setup.

jokase · 2018-12-15T02:11:29Z

I think the root cause is issue 1745
...edit - pretty much confirmed it is the root cause by attempting to build thinc/thinc-gpu-ops manually.
You get the build error mentioned by @max0xff (6 Feb):

error: Don't know how to compile include/_cuda_shim.cu to build\temp.win-amd64-3.6\Release\include/_cuda_shim.obj

honnibal · 2018-12-16T17:14:00Z

Just released v0.0.4 of thinc_gpu_ops, which should resolve the incompatibility with Python 3.7. I think this should resolve the outstanding problems. Thanks for your patience on this -- unfortunately we can't test these functions with the CI (as it doesn't have GPU), so it's hard to ensure cross-platform errors don't creep in.

geg00 · 2019-02-05T00:41:48Z

I'm trying to use it with CUDA 10.0
pip install spacy[cuda100]
But with Windows 10
python
Python 3.7.1 (default, Dec 10 2018, 22:54:23) [MSC v.1915 64 bit (AMD64)] :: Anaconda, Inc. on win32
Type "help", "copyright", "credits" or "license" for more information.

import spacy
spacy.version
'2.0.18'
nlp = spacy.load('en')
doc = nlp('This is great')

import spacy
spacy.prefer_gpu()
True
nlp = spacy.load('en')
doc = nlp('My life is great')
Traceback (most recent call last):
File "", line 1, in
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\spacy\language.py", line 346, in call
doc = proc(doc)
File "pipeline.pyx", line 426, in spacy.pipeline.Tagger.call
File "pipeline.pyx", line 438, in spacy.pipeline.Tagger.predict
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\neural_classes\model.py", line 161, in call
return self.predict(x)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 55, in predict
X = layer(X)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\neural_classes\model.py", line 161, in call
return self.predict(x)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 293, in predict
X = layer(layer.ops.flatten(seqs_in, pad=pad))
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\neural_classes\model.py", line 161, in call
return self.predict(x)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 55, in predict
X = layer(X)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\neural_classes\model.py", line 161, in call
return self.predict(x)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\neural_classes\model.py", line 125, in predict
y, _ = self.begin_update(X)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 374, in uniqued_fwd
Y_uniq, bp_Y_uniq = layer.begin_update(X_uniq, drop=drop)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 61, in begin_update
X, inc_layer_grad = layer.begin_update(X, drop=drop)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 176, in begin_update
values = [fwd(X, *a, **k) for fwd in forward]
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 176, in
values = [fwd(X, *a, **k) for fwd in forward]
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 258, in wrap
output = func(*args, **kwargs)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 176, in begin_update
values = [fwd(X, *a, **k) for fwd in forward]
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 176, in
values = [fwd(X, *a, **k) for fwd in forward]
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 258, in wrap
output = func(*args, **kwargs)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 176, in begin_update
values = [fwd(X, *a, **k) for fwd in forward]
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 176, in
values = [fwd(X, *a, **k) for fwd in forward]
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 258, in wrap
output = func(*args, **kwargs)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\neural_classes\hash_embed.py", line 51, in begin_update
keys = self.ops.hash(ids, self.seed) % self.nV
File "ops.pyx", line 821, in thinc.neural.ops.CupyOps.hash
AttributeError: module 'thinc_gpu_ops' has no attribute 'hash'

????

File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\spacy\language.py", line 346, in call
doc = proc(doc)
File "pipeline.pyx", line 426, in spacy.pipeline.Tagger.call
File "pipeline.pyx", line 438, in spacy.pipeline.Tagger.predict
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\neural_classes\model.py", line 161, in call
return self.predict(x)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 55, in predict
X = layer(X)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\neural_classes\model.py", line 161, in call
return self.predict(x)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 293, in predict
X = layer(layer.ops.flatten(seqs_in, pad=pad))
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\neural_classes\model.py", line 161, in call
return self.predict(x)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 55, in predict
X = layer(X)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\neural_classes\model.py", line 161, in call
return self.predict(x)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\neural_classes\model.py", line 125, in predict
y, _ = self.begin_update(X)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 374, in uniqued_fwd
Y_uniq, bp_Y_uniq = layer.begin_update(X_uniq, drop=drop)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 61, in begin_update
X, inc_layer_grad = layer.begin_update(X, drop=drop)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 176, in begin_update
values = [fwd(X, *a, **k) for fwd in forward]
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 176, in
values = [fwd(X, *a, **k) for fwd in forward]
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 258, in wrap
output = func(*args, **kwargs)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 176, in begin_update
values = [fwd(X, *a, **k) for fwd in forward]
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 176, in
values = [fwd(X, *a, **k) for fwd in forward]
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 258, in wrap
output = func(*args, **kwargs)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 176, in begin_update
values = [fwd(X, *a, **k) for fwd in forward]
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 176, in
values = [fwd(X, *a, **k) for fwd in forward]
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\api.py", line 258, in wrap
output = func(*args, **kwargs)
File "C:\Users\hbb9279\AppData\Local\Continuum\anaconda3\lib\site-packages\thinc\neural_classes\hash_embed.py", line 51, in begin_update
keys = self.ops.hash(ids, self.seed) % self.nV
File "ops.pyx", line 821, in thinc.neural.ops.CupyOps.hash
AttributeError: module 'thinc_gpu_ops' has no attribute 'hash'

honnibal · 2019-02-05T01:25:39Z

@gerardogarciag1 The error means the library hasn't been compiled properly. Do you have a compiler installed?

It may be easier to debug if you clone the https://github.com/explosion/thinc_gpu_ops repository and try to build it explicitly.

geg00 · 2019-02-05T03:07:03Z

I do not have compiler
I ran the pip uninstall thinc-gpu-ops

I ran the installation again
(base) C:\Users\hbb9279\1.0\thinc_gpu_ops>pip install -U thinc-gpu-ops
Collecting thinc-gpu-ops
Requirement already satisfied, skipping upgrade: numpy>=1.7.0 in c:\users\hbb9279\appdata\local\continuum\anaconda3\lib\site-packages (from thinc-gpu-ops) (1.15.4)
Installing collected packages: thinc-gpu-ops
Successfully installed thinc-gpu-ops-0.0.4
No success :(

Any other suggestions?

farleylai · 2019-02-06T23:30:12Z

Same error as @gerardogarciag1 but in a python 3.6 conda env on ubuntu 16.04.
Installation of thinc-gpu-ops-0.0.4 as SpaCy[cuda100] dependency.

Spacy recognizes the gpu availability but thinc_gpu_ops seems failing to import the Cython module in hinc_gpu_ops/thinc_gpu_ops/__init__.py.

>>> import spacy
>>> spacy.prefer_gpu()
True
>>> import thinc_gpu_ops
>>> thinc_gpu_ops.AVAILABLE
False

I further looked into the package installation path and found gpu_ops.pyx but it seems like no compiled cython there.

(py36) $ ll ~/miniconda3/envs/sinet36/lib/python3.6/site-packages/thinc_gpu_ops/
-rw-r--r-- 1 farleylai ML    443 Feb  6 18:31 about.py
-rw-r--r-- 1 farleylai ML 419404 Feb  6 18:31 gpu_ops.cpp
-rw-r--r-- 1 farleylai ML   4150 Feb  6 18:31 gpu_ops.pyx
-rw-r--r-- 1 farleylai ML    126 Feb  6 18:31 __init__.py
drwxr-xr-x 2 farleylai ML   4096 Feb  6 18:46 __pycache__/

UPDATE

It seems like the compilation is not triggered by default.
pip install thinc-gpu-ops does not produce the compiled Cython module.
Reinstalling as follows finally gets the Cython module compiled to work:

pip install --force-reinstall --no-binary :all: thinc-gpu-ops

(py36) $ ll ~/miniconda3/envs/sinet36/lib/python3.6/site-packages/thinc_gpu_ops/
-rw-r--r-- 1 farleylai ML    443 Dec 16 12:01 about.py
-rw-r--r-- 1 farleylai ML 419404 Dec 16 12:00 gpu_ops.cpp
-rwxr-xr-x 1 farleylai ML 507256 Feb  6 19:21 gpu_ops.cpython-36m-x86_64-linux-gnu.so*
-rw-r--r-- 1 farleylai ML   4150 Dec 16 09:50 gpu_ops.pyx
-rw-r--r-- 1 farleylai ML    126 Dec 16 09:50 __init__.py

mishraprafful · 2019-03-11T06:30:38Z

facing the same issue .....rebuilt thinc_gpu_ops but it doesn't seem to help. @honnibal

erotavlas · 2019-03-27T20:41:00Z

I'm also having this issue

AttributeError: module 'thinc_gpu_ops' has no attribute 'hash'

Python 3.6.8 |Anaconda, Inc.| (default, Feb 21 2019, 18:30:04) [MSC v.1916 64 bit (AMD64)] on win32
Type "help", "copyright", "credits" or "license" for more information.
>>> import spacy
>>> spacy.prefer_gpu()
True
>>> import thinc_gpu_ops
>>> thinc_gpu_ops.AVAILABLE
False
>>>

I tried the command mentioned above

pip install --force-reinstall --no-binary :all: thinc-gpu-ops

but it did not help

mathematiguy · 2019-03-30T00:29:39Z

I hit the same problem. I have a 2080Ti, and used the following Dockerfile, which I built in Ubuntu 18.04:

FROM nvidia/cuda:10.0-cudnn7-runtime-ubuntu18.04

RUN apt update && apt install -y build-essential python3-dev python3-pip

RUN pip3 install -U spacy[cuda100]
RUN python3 -m spacy download en_core_web_sm

Note python3-dev and build-essential are installed. I want to run spacy on python3, but I also included python-dev (as well as python3-dev) in a separate run, just to see if it would help, but it did not.

>>> import spacy
>>> spacy.__version__
'2.0.18'
>>> spacy.prefer_gpu()
True
>>> nlp = spacy.load('en_core_web_sm')
>>> doc = nlp('My life is great')
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python3.6/dist-packages/spacy/language.py", line 385, in __call__
    doc = proc(doc, **component_cfg.get(name, {}))
  File "pipes.pyx", line 374, in spacy.pipeline.pipes.Tagger.__call__
  File "pipes.pyx", line 393, in spacy.pipeline.pipes.Tagger.predict
  File "/usr/local/lib/python3.6/dist-packages/thinc/neural/_classes/model.py", line 165, in __call__
    return self.predict(x)
  File "/usr/local/lib/python3.6/dist-packages/thinc/neural/_classes/feed_forward.py", line 40, in predict
    X = layer(X)
  File "/usr/local/lib/python3.6/dist-packages/thinc/neural/_classes/model.py", line 165, in __call__
    return self.predict(x)
  File "/usr/local/lib/python3.6/dist-packages/thinc/api.py", line 279, in predict
    X = layer(layer.ops.flatten(seqs_in, pad=pad))
  File "/usr/local/lib/python3.6/dist-packages/thinc/neural/_classes/model.py", line 165, in __call__
    return self.predict(x)
  File "/usr/local/lib/python3.6/dist-packages/thinc/neural/_classes/feed_forward.py", line 40, in predict
    X = layer(X)
  File "/usr/local/lib/python3.6/dist-packages/thinc/neural/_classes/model.py", line 165, in __call__
    return self.predict(x)
  File "/usr/local/lib/python3.6/dist-packages/thinc/neural/_classes/model.py", line 129, in predict
    y, _ = self.begin_update(X, drop=None)
  File "/usr/local/lib/python3.6/dist-packages/thinc/api.py", line 348, in uniqued_fwd
    Y_uniq, bp_Y_uniq = layer.begin_update(X_uniq, drop=drop)
  File "/usr/local/lib/python3.6/dist-packages/thinc/neural/_classes/feed_forward.py", line 46, in begin_update
    X, inc_layer_grad = layer.begin_update(X, drop=drop)
  File "/usr/local/lib/python3.6/dist-packages/thinc/api.py", line 132, in begin_update
    values = [fwd(X, *a, **k) for fwd in forward]
  File "/usr/local/lib/python3.6/dist-packages/thinc/api.py", line 132, in <listcomp>
    values = [fwd(X, *a, **k) for fwd in forward]
  File "/usr/local/lib/python3.6/dist-packages/thinc/api.py", line 225, in wrap
    output = func(*args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/thinc/api.py", line 132, in begin_update
    values = [fwd(X, *a, **k) for fwd in forward]
  File "/usr/local/lib/python3.6/dist-packages/thinc/api.py", line 132, in <listcomp>
    values = [fwd(X, *a, **k) for fwd in forward]
  File "/usr/local/lib/python3.6/dist-packages/thinc/api.py", line 225, in wrap
    output = func(*args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/thinc/api.py", line 132, in begin_update
    values = [fwd(X, *a, **k) for fwd in forward]
  File "/usr/local/lib/python3.6/dist-packages/thinc/api.py", line 132, in <listcomp>
    values = [fwd(X, *a, **k) for fwd in forward]
  File "/usr/local/lib/python3.6/dist-packages/thinc/api.py", line 225, in wrap
    output = func(*args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/thinc/neural/_classes/hash_embed.py", line 55, in begin_update
    keys = self.ops.hash(ids, self.seed) % self.nV
  File "ops.pyx", line 917, in thinc.neural.ops.CupyOps.hash
AttributeError: module 'thinc_gpu_ops' has no attribute 'hash'

I also checked my path variables, following @honnibal's advice

# checking if LD_LIBRARY_PATH contains /usr/local/cuda/lib64 - it does
echo $LD_LIBRARY_PATH
/usr/local/nvidia/lib:/usr/local/nvidia/lib64

# checking if PATH contains /usr/local/cuda/bin - it does
echo $PATH
/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

Happy to do more tests if they might be useful. Let me know if I can help.

DenisBlondeel · 2019-04-09T12:28:52Z

@honnibal
I have exactly the same issues and errors as the above. Here are my specs:

pip freeze
cupy-cuda100==5.4.0
en-core-web-sm==2.1.0
face-recognition-models==0.3.0
numpy==1.16.2
spacy==2.1.3
thinc==7.0.4
thinc-gpu-ops==0.0.4

and i have for my cuda toolkit

nvcc -V
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2018 NVIDIA Corporation
Built on Sat_Aug_25_21:08:04_Central_Daylight_Time_2018
Cuda compilation tools, release 10.0, V10.0.130

and everytime I run my application with the
spacy.require_gpu()

i get the following

Traceback (most recent call last):
  File "D:/Users/r0372092/Stage_TI/SpaCy/eu/xenit/spacy/main.py", line 12, in <module>
    doc = nlp(myfile)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\spacy\language.py", line 385, in __call__
    doc = proc(doc, **component_cfg.get(name, {}))
  File "pipes.pyx", line 374, in spacy.pipeline.pipes.Tagger.__call__
  File "pipes.pyx", line 393, in spacy.pipeline.pipes.Tagger.predict
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\neural\_classes\model.py", line 165, in __call__
    return self.predict(x)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\neural\_classes\feed_forward.py", line 40, in predict
    X = layer(X)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\neural\_classes\model.py", line 165, in __call__
    return self.predict(x)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\api.py", line 279, in predict
    X = layer(layer.ops.flatten(seqs_in, pad=pad))
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\neural\_classes\model.py", line 165, in __call__
    return self.predict(x)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\neural\_classes\feed_forward.py", line 40, in predict
    X = layer(X)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\neural\_classes\model.py", line 165, in __call__
    return self.predict(x)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\neural\_classes\model.py", line 129, in predict
    y, _ = self.begin_update(X, drop=None)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\api.py", line 348, in uniqued_fwd
    Y_uniq, bp_Y_uniq = layer.begin_update(X_uniq, drop=drop)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\neural\_classes\feed_forward.py", line 46, in begin_update
    X, inc_layer_grad = layer.begin_update(X, drop=drop)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\api.py", line 132, in begin_update
    values = [fwd(X, *a, **k) for fwd in forward]
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\api.py", line 132, in <listcomp>
    values = [fwd(X, *a, **k) for fwd in forward]
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\api.py", line 225, in wrap
    output = func(*args, **kwargs)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\api.py", line 132, in begin_update
    values = [fwd(X, *a, **k) for fwd in forward]
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\api.py", line 132, in <listcomp>
    values = [fwd(X, *a, **k) for fwd in forward]
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\api.py", line 225, in wrap
    output = func(*args, **kwargs)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\api.py", line 132, in begin_update
    values = [fwd(X, *a, **k) for fwd in forward]
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\api.py", line 132, in <listcomp>
    values = [fwd(X, *a, **k) for fwd in forward]
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\api.py", line 225, in wrap
    output = func(*args, **kwargs)
  File "C:\Users\r0372092\AppData\Local\Programs\Python\Python36\lib\site-packages\thinc\neural\_classes\hash_embed.py", line 55, in begin_update
    keys = self.ops.hash(ids, self.seed) % self.nV
  File "ops.pyx", line 917, in thinc.neural.ops.CupyOps.hash
AttributeError: module 'thinc_gpu_ops' has no attribute 'hash'
Traceback (most recent call last):
  File "cupy\cuda\memory.pyx", line 596, in cupy.cuda.memory.PooledMemory.free
  File "cupy\cuda\memory.pyx", line 965, in cupy.cuda.memory.SingleDeviceMemoryPool.free
  File "cupy\cuda\memory.pyx", line 787, in cupy.cuda.memory.LockAndNoGc.__enter__
AttributeError: 'NoneType' object has no attribute 'isenabled'

alexookah · 2019-04-15T15:31:15Z

@honnibal why is this issue closed? the same issue still exists
check here also: explosion/spaCy#3554

i have the same issue by the way

devforfu · 2019-05-06T12:16:06Z

Have the same error on Ubuntu with latest pip version of spaCy. There are some references on how to solve it but none of them helps to fix my case. Other GPU-dependant libraries work well (i.e. PyTorch, TensorFlow, Catboost, etc.).

Could you please create some README with the most recent recommendations on how to enable GPU training? Right now it is a bit difficult to make spaCy work with GPU.

bklippstein · 2019-05-25T15:33:01Z

I have the same problem.
I did pip install --force-reinstall --no-binary :all: thinc-gpu-ops

After this it's better:
import thinc_gpu_ops
thinc_gpu_ops.AVAILABLE
>> True

Before the reinstall it was False.
But i still get AttributeError: module 'thinc_gpu_ops' has no attribute 'hash'

tomstelk · 2019-06-05T13:35:01Z

I am having the same issue on Windows 7.

Cuda 9.0:

(base) C:\Users\admin>nvcc --version
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2017 NVIDIA Corporation
Built on Fri_Sep__1_21:08:32_Central_Daylight_Tim
Cuda compilation tools, release 9.0, V9.0.176

Python 3.7

(base) C:\Users\admin>python --version
Python 3.7.3

Successfully run spacy install with cuda for the correct version:
pip install -U spacy[cuda90]

Test if cupy is working:

>>> import cupy
>>>
>>>
>>> x=cupy.array([1,2,3])
>>> 2*x
array([2, 4, 6], dtype=int32)

Test thinc-gpu-ops:

>>> import thinc_gpu_ops
>>> thinc_gpu_ops.AVAILABLE
False
>>>

Uninstall thinc-gpu-ops and reinstall with the command above:


(base) C:\Users\Public\Documents\Cassandre\Stage 2 - Transform\S2-Transform>pip
install --force-reinstall --no-binary :all: thinc-gpu-ops
Collecting thinc-gpu-ops
  Using cached https://files.pythonhosted.org/packages/a4/ad/11ab80a24bcedd7dd0c
fabaedba2ceaeca11f1aaeeff432a3d2e63ca7d02/thinc_gpu_ops-0.0.4.tar.gz
Collecting numpy>=1.7.0 (from thinc-gpu-ops)
  Using cached https://files.pythonhosted.org/packages/d3/4b/f9f4b96c0b1ba43d28a
5bdc4b64f0b9d3fbcf31313a51bc766942866a7c7/numpy-1.16.4.zip
Skipping bdist_wheel for thinc-gpu-ops, due to binaries being disabled for it.
Skipping bdist_wheel for numpy, due to binaries being disabled for it.
tables 3.5.1 requires mock>=2.0, which is not installed.
Installing collected packages: numpy, thinc-gpu-ops
  Found existing installation: numpy 1.16.4
    Uninstalling numpy-1.16.4:
      Successfully uninstalled numpy-1.16.4
  Running setup.py install for numpy ... done
  Running setup.py install for thinc-gpu-ops ... done
Successfully installed numpy-1.16.4 thinc-gpu-ops-0.0.4

Test thinc-gpu-ops one last time and Failed:

>>> import thinc_gpu_ops
>>> thinc_gpu_ops.AVAILABLE
False
>>>

tomstelk · 2019-06-06T14:37:19Z

@honnibal - Might the locate_cuda function in setup.py be incorrect for windows?

locate_cuda looks like:


def locate_cuda():
    """Locate the CUDA environment on the system

    Returns a dict with keys 'home', 'nvcc', 'include', and 'lib64'
    and values giving the absolute path to each directory.

    Starts by looking for the CUDAHOME env variable. If not found, everything
    is based on finding 'nvcc' in the PATH.
    """

    # first check if the CUDAHOME env variable is in use
    if 'CUDA_HOME' in os.environ:
        home = os.environ['CUDA_HOME']
        nvcc = os.path.join(home, 'bin', 'nvcc')
    elif os.path.exists('/usr/local/cuda/bin/nvcc'):
        home = '/usr/local/cuda'
        nvcc = os.path.join(home, 'bin', 'nvcc')
    else:
        # otherwise, search the PATH for NVCC
        nvcc = find_in_path('nvcc', os.environ['PATH'])
        if nvcc is None:
            print('Warning: The nvcc binary could not be located in your $PATH. '
                  'For GPU capability, either add it to your path, or set $CUDA_HOME')
            return None
        home = os.path.dirname(os.path.dirname(nvcc))

    cudaconfig = {'home':home, 'nvcc':nvcc,
                  'include': os.path.join(home, 'include'),
                  'lib64': os.path.join(home, 'lib64')}
    for k, v in cudaconfig.items():
        if not os.path.exists(v):
            print('Warning: The CUDA %s path could not be located in %s' % (k, v))
            return None
    return cudaconfig

Doesn't look this reflects the directory structure of CUDA_HOME on Windows?

Firstly the value for nvcc in the config should be: os.path.join(home, 'bin', 'nvcc.exe')
And there is no lib64 directory, instead should be os.path.join(home, 'lib', 'x64')

lw3259111 · 2019-06-23T03:51:20Z

I have the same problem.
I did pip install --force-reinstall --no-binary :all: thinc-gpu-ops

but still no resolve
import thinc_gpu_ops
thinc_gpu_ops.AVAILABLE

False

pythonBerg · 2019-08-07T22:30:12Z

Has anyone resolved this? I first filed this issue in October but this thread is more lively.

asevans48 · 2019-08-23T04:35:40Z

Still an issue on Windows 10. Same as above for Cuda 10.1 and cupy-cuda101

mmunozm · 2019-08-27T21:39:36Z

I think that the problem might be with the setup.py file for thinc_gpu_ops, especially for Windows but might also affect Linux distributions (at least in the github accessible 0.0.3 version)
https://github.com/explosion/thinc_gpu_ops/blob/master/setup.py#L128-L130

In my latest CUDA installation (10.1), the CUDA_HOME variable doesn't exist. I have CUDA_PATH and CUDA_PATH_V10_1.

If you create that variable and add your CUDA installation directory, the lack of an ".exe" extension makes the CUDA variable None
https://github.com/explosion/thinc_gpu_ops/blob/master/setup.py#L146-L149
https://github.com/explosion/thinc_gpu_ops/blob/master/setup.py#L153

That will later prevent the build of gpu_ops module
https://github.com/explosion/thinc_gpu_ops/blob/master/setup.py#L203-L207

The solution will be download the source, adapt CUDA_HOME to CUDA_PATH (or create a CUDA_HOME variable), and change nvcc to nvcc.exe for Windows compilation.

I will try it tomorrow and let you know.

pythonBerg · 2019-08-27T21:59:59Z

I think there is a little more to it. I've attempted each/all of these changes and am still hitting a wall. Good luck. I'd love to break through this wall...

…

On Tue, Aug 27, 2019 at 5:39 PM Miguel Muñoz ***@***.***> wrote: I think that the problem might be with the setup.py file for thinc_gpu_ops, especially for Windows but might also affect Linux distributions (at least in the github accessible 0.0.3 version) https://github.com/explosion/thinc_gpu_ops/blob/master/setup.py#L128-L130 In my latest CUDA installation (10.1), the CUDA_HOME variable doesn't exist. I have CUDA_PATH and CUDA_PATH_V10_1. If you create that variable and add your CUDA installation directory, the lack of an ".exe" extension makes the CUDA variable None https://github.com/explosion/thinc_gpu_ops/blob/master/setup.py#L146-L149 https://github.com/explosion/thinc_gpu_ops/blob/master/setup.py#L153 That will later prevent the build of gpu_ops module https://github.com/explosion/thinc_gpu_ops/blob/master/setup.py#L203-L207 The solution will be download the source, adapt CUDA_HOME to CUDA_PATH (or create a CUDA_HOME variable), and change nvcc to nvcc.exe for Windows compilation. I will try it tomorrow and let you know. — You are receiving this because you commented. Reply to this email directly, view it on GitHub <#79?email_source=notifications&email_token=AFOVOMNNQWCJ63PHG6322ELQGWNJ3A5CNFSM4F7JCO3KYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOD5JGNUQ#issuecomment-525493970>, or mute the thread <https://github.com/notifications/unsubscribe-auth/AFOVOMLFLAMGV5UIF3DOLCTQGWNJ3ANCNFSM4F7JCO3A> .

mtmiron · 2019-08-31T01:41:46Z

@pythonBerg @mmunozm To save you some time: #92 (comment)

SergiiShcherbak · 2019-11-27T00:44:35Z

This solution worked for me: https://www.gitmemory.com/issue/explosion/spaCy/3682/489917991

geg00 · 2019-11-27T11:16:50Z

Just update to the latest release and it's fixed there.
https://spacy.io/usage#gpu
👍

honnibal closed this as completed Dec 16, 2018

ghost mentioned this issue Feb 9, 2019

thinc_gpu_ops not built properly #92

Closed

BramVanroy mentioned this issue Nov 29, 2019

Unexpected (almost random) parses and no sentence segmentation when using GPU on Windows explosion/spaCy#4724

Closed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

module 'thinc_gpu_ops' has no attribute 'hash' #79

module 'thinc_gpu_ops' has no attribute 'hash' #79

j2l commented Oct 25, 2018

pavanAlluri commented Oct 29, 2018

folterj commented Nov 1, 2018

ZackAttack614 commented Nov 3, 2018

honnibal commented Nov 4, 2018

folterj commented Nov 12, 2018

honnibal commented Dec 2, 2018

honnibal commented Dec 2, 2018 •

edited

Loading

honnibal commented Dec 2, 2018

0x7r1574n commented Dec 15, 2018

jokase commented Dec 15, 2018 •

edited

Loading

honnibal commented Dec 16, 2018

geg00 commented Feb 5, 2019 •

edited

Loading

honnibal commented Feb 5, 2019

geg00 commented Feb 5, 2019

farleylai commented Feb 6, 2019 •

edited

Loading

mishraprafful commented Mar 11, 2019

erotavlas commented Mar 27, 2019 •

edited

Loading

mathematiguy commented Mar 30, 2019 •

edited

Loading

DenisBlondeel commented Apr 9, 2019

alexookah commented Apr 15, 2019 •

edited

Loading

devforfu commented May 6, 2019

bklippstein commented May 25, 2019

tomstelk commented Jun 5, 2019

tomstelk commented Jun 6, 2019

lw3259111 commented Jun 23, 2019 •

edited

Loading

pythonBerg commented Aug 7, 2019

asevans48 commented Aug 23, 2019

mmunozm commented Aug 27, 2019

pythonBerg commented Aug 27, 2019 via email

mtmiron commented Aug 31, 2019

SergiiShcherbak commented Nov 27, 2019

geg00 commented Nov 27, 2019

module 'thinc_gpu_ops' has no attribute 'hash' #79

module 'thinc_gpu_ops' has no attribute 'hash' #79

Comments

j2l commented Oct 25, 2018

pavanAlluri commented Oct 29, 2018

folterj commented Nov 1, 2018

ZackAttack614 commented Nov 3, 2018

honnibal commented Nov 4, 2018

folterj commented Nov 12, 2018

honnibal commented Dec 2, 2018

honnibal commented Dec 2, 2018 • edited Loading

honnibal commented Dec 2, 2018

0x7r1574n commented Dec 15, 2018

jokase commented Dec 15, 2018 • edited Loading

honnibal commented Dec 16, 2018

geg00 commented Feb 5, 2019 • edited Loading

honnibal commented Feb 5, 2019

geg00 commented Feb 5, 2019

farleylai commented Feb 6, 2019 • edited Loading

mishraprafful commented Mar 11, 2019

erotavlas commented Mar 27, 2019 • edited Loading

mathematiguy commented Mar 30, 2019 • edited Loading

DenisBlondeel commented Apr 9, 2019

alexookah commented Apr 15, 2019 • edited Loading

devforfu commented May 6, 2019

bklippstein commented May 25, 2019

tomstelk commented Jun 5, 2019

tomstelk commented Jun 6, 2019

lw3259111 commented Jun 23, 2019 • edited Loading

pythonBerg commented Aug 7, 2019

asevans48 commented Aug 23, 2019

mmunozm commented Aug 27, 2019

pythonBerg commented Aug 27, 2019 via email

mtmiron commented Aug 31, 2019

SergiiShcherbak commented Nov 27, 2019

geg00 commented Nov 27, 2019

honnibal commented Dec 2, 2018 •

edited

Loading

jokase commented Dec 15, 2018 •

edited

Loading

geg00 commented Feb 5, 2019 •

edited

Loading

farleylai commented Feb 6, 2019 •

edited

Loading

erotavlas commented Mar 27, 2019 •

edited

Loading

mathematiguy commented Mar 30, 2019 •

edited

Loading

alexookah commented Apr 15, 2019 •

edited

Loading

lw3259111 commented Jun 23, 2019 •

edited

Loading