Sample cod Error on Gpu: RuntimeError: cublasSgemm(dev.cublas_handle,

[yang182]

import numpy as np
import dynet_config
dynet_config.set_gpu()

import dynet as dy
from optparse import OptionParser
parser = OptionParser()
parser.add_option("--dynet-gpu", action="store_true")
class OurNetwork(object):
def init(self, pc):
self.pW = pc.add_parameters((10,30))
self.pB = pc.add_parameters(10)
self.lookup = pc.add_lookup_parameters((500,10))

def __call__(self, inputs):
    lookup = self.lookup
    emb_vectors = [lookup[i] for i in inputs]
    net_input = dy.concatenate(emb_vectors)
    net_output = dy.softmax((self.pW * net_input) + self.pB)
    return net_output

def create_network_return_loss(self, inputs, expected_output):
    dy.renew_cg()
    out = self(inputs)
    loss = -dy.log(dy.pick(out, expected_output))
    return loss

def create_network_return_best(self, inputs):
    dy.renew_cg()
    out = self(inputs)
    return np.argmax(out.npvalue())

dy.init()
dy.renew_cg()

m = dy.Model()

network = OurNetwork(m)

trainer = dy.SimpleSGDTrainer(m)

for epoch in range(50):
for inp,lbl in (([1,2,3],1), ([3,2,4],2)):
loss = network.create_network_return_loss(inp, lbl)
loss.value()
loss.backward()
trainer.update()
print(loss.value()) # need to run loss.value() for the forward prop

print('Predicted smallest element among {} is {}:'.format([1,2,3], network.create_network_return_best([1,2,3])))

==========================out info=================
[dynet] initializing CUDA
[dynet] CUDA driver/runtime versions are 10.0/10.0
Request for 1 GPU ...
[dynet] Device Number: 0
[dynet] Device name: GeForce RTX 2080 Ti
[dynet] Memory Clock Rate (KHz): 7000000
[dynet] Memory Bus Width (bits): 352
[dynet] Peak Memory Bandwidth (GB/s): 616
[dynet] Memory Free (GB): 1.65767/11.5229
[dynet]
[dynet] Device(s) selected: 0
[dynet] random seed: 3237171193
[dynet] allocating memory: 512MB
[dynet] memory allocation done.
WARNING: Attempting to initialize dynet twice. Ignoring duplicate initialization.
CUBLAS failure in cublasSgemm(dev.cublas_handle, CUBLAS_OP_N, CUBLAS_OP_N, y.d.rows(), y.d.cols() * y.d.batch_elems(), l.d.cols(), dev.kSCALAR_ONE, l.v, l.d.rows(), r.v, r.d.rows(), acc_scalar, y.v, y.d.rows())
13
Traceback (most recent call last):
File "/data/jupyter/hongchao/text2structure/model2note/model/test_dy_gpu.py", line 57, in
loss.value()
File "_dynet.pyx", line 769, in _dynet.Expression.value
File "_dynet.pyx", line 783, in _dynet.Expression.value
RuntimeError: cublasSgemm(dev.cublas_handle, CUBLAS_OP_N, CUBLAS_OP_N, y.d.rows(), y.d.cols() * y.d.batch_elems(), l.d.cols(), dev.kSCALAR_ONE, l.v, l.d.rows(), r.v, r.d.rows(), acc_scalar, y.v, y.d.rows())

Process finished with exit code 1

[cydur]

I've got a similar problem using C++ with CUDA V 10.2.
In my case the forward step works fine and I get a loss value which looks OK. But in the backward step there is an Exception telling:
** On entry to SGEMM parameter number 8 had an illegal value
CUBLAS failure in cublasSgemm(dev.cublas_handle, CUBLAS_OP_N, CUBLAS_OP_T, y.d.rows(), y.d.cols(), l.d.cols() * l.d.batch_elems(), dev.kSCALAR_ONE, l.v, l.d.rows(), r.v, r.d.rows(), dev.kSCALAR_ONE, y.v, y.d.rows())
7

This only occurs with dynet-autobatch turned on. Without autobatch it works fine.

Any ideas someone?