Type Error launching kernel on GPU

[Scott-Wenzel]

Hello - Trying to run mandelbrot_numba.ipynb but I cannot launch kernel on GPU. Error follows as well as info about my environment at end. Thanks! Scott Wenzel

Error

TypingErrorTraceback (most recent call last)
<ipython-input-10-ce3dd5de1d09> in <module>()
      5 start = timer()
      6 d_image = cuda.to_device(gimage)
----> 7 mandel_kernel[griddim, blockdim](-2.0, 1.0, -1.0, 1.0, d_image, 20)
      8 d_image.to_host()
      9 dt = timer() - start

/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.pyc in __call__(self, *args)
    699         Specialize and invoke this kernel with *args*.
    700         '''
--> 701         kernel = self.specialize(*args)
    702         cfg = kernel[self.griddim, self.blockdim, self.stream, self.sharedmem]
    703         cfg(*args)

/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.pyc in specialize(self, *args)
    715                 self.targetoptions['link'] = ()
    716             kernel = compile_kernel(self.py_func, argtypes,
--> 717                                     **self.targetoptions)
    718             self.definitions[argtypes] = kernel
    719             if self.bind:

/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.pyc in core(*args, **kwargs)
     35     def core(*args, **kwargs):
     36         with _cuda_compiler_lock:
---> 37             return fn(*args, **kwargs)
     38     return core
     39 

/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.pyc in compile_kernel(pyfunc, args, link, debug, inline, fastmath)
     73 def compile_kernel(pyfunc, args, link, debug=False, inline=False,
     74                    fastmath=False):
---> 75     cres = compile_cuda(pyfunc, types.void, args, debug=debug, inline=inline)
     76     fname = cres.fndesc.llvm_func_name
     77     lib, kernel = cres.target_context.prepare_cuda_kernel(cres.library, fname,

/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.pyc in core(*args, **kwargs)
     35     def core(*args, **kwargs):
     36         with _cuda_compiler_lock:
---> 37             return fn(*args, **kwargs)
     38     return core
     39 

/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.pyc in compile_cuda(pyfunc, return_type, args, debug, inline)
     62                                   return_type=return_type,
     63                                   flags=flags,
---> 64                                   locals={})
     65 
     66     library = cres.library

/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.pyc in compile_extra(typingctx, targetctx, func, args, return_type, flags, locals, library)
    691     pipeline = Pipeline(typingctx, targetctx, library,
    692                         args, return_type, flags, locals)
--> 693     return pipeline.compile_extra(func)
    694 
    695 

/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.pyc in compile_extra(self, func)
    348         self.lifted = ()
    349         self.lifted_from = None
--> 350         return self._compile_bytecode()
    351 
    352     def compile_ir(self, func_ir, lifted=(), lifted_from=None):

/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.pyc in _compile_bytecode(self)
    656         """
    657         assert self.func_ir is None
--> 658         return self._compile_core()
    659 
    660     def _compile_ir(self):

/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.pyc in _compile_core(self)
    643 
    644         pm.finalize()
--> 645         res = pm.run(self.status)
    646         if res is not None:
    647             # Early pipeline completion

/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.pyc in run(self, status)
    234                     # No more fallback pipelines?
    235                     if is_final_pipeline:
--> 236                         raise patched_exception
    237                     # Go to next fallback pipeline
    238                     else:

TypingError: Caused By:
Traceback (most recent call last):
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 228, in run
    stage()
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 442, in stage_nopython_frontend
    self.locals)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 794, in type_inference_stage
    infer.propagate()
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typeinfer.py", line 767, in propagate
    raise errors[0]
TypingError: Internal error at <numba.typeinfer.CallConstraint object at 0x7f3fc80d5790>:
--%<-----------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typeinfer.py", line 128, in propagate
    constraint(typeinfer)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typeinfer.py", line 379, in __call__
    self.resolve(typeinfer, typevars, fnty)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/contextlib.py", line 35, in __exit__
    self.gen.throw(type, value, traceback)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/errors.py", line 249, in new_error_context
    six.reraise(type(newerr), newerr, sys.exc_info()[2])
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/errors.py", line 243, in new_error_context
    yield
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typeinfer.py", line 379, in __call__
    self.resolve(typeinfer, typevars, fnty)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typeinfer.py", line 392, in resolve
    sig = typeinfer.resolve_call(fnty, pos_args, kw_args)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typeinfer.py", line 1030, in resolve_call
    return self.context.resolve_function_type(fnty, pos_args, kw_args)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typing/context.py", line 202, in resolve_function_type
    return func.get_call_type(self, args, kws)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/types/functions.py", line 49, in get_call_type
    sig = temp.apply(args, kws)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typing/templates.py", line 185, in apply
    sig = generic(args, kws)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.py", line 151, in generic
    return dft.compile(args)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.py", line 120, in compile
    inline=self.inline)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.py", line 37, in core
    return fn(*args, **kwargs)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.py", line 64, in compile_cuda
    locals={})
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 693, in compile_extra
    return pipeline.compile_extra(func)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 350, in compile_extra
    return self._compile_bytecode()
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 658, in _compile_bytecode
    return self._compile_core()
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 645, in _compile_core
    res = pm.run(self.status)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 236, in run
    raise patched_exception
InternalError: Caused By:
Traceback (most recent call last):
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 228, in run
    stage()
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 364, in stage_analyze_bytecode
    func_ir = translate_stage(self.func_id, self.bc)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 756, in translate_stage
    return interp.interpret(bytecode)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/interpreter.py", line 109, in interpret
    self._dispatch(inst, kws)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/interpreter.py", line 229, in _dispatch
    return fn(inst, **kws)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/interpreter.py", line 566, in op_LOAD_GLOBAL
    value = self.get_global_value(name)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/interpreter.py", line 185, in get_global_value
    return utils.get_function_globals(self.func_id.func)[name]
AttributeError: 'CPUDispatcher' object has no attribute 'func_globals'

Failed at nopython (analyzing bytecode)
'CPUDispatcher' object has no attribute 'func_globals'
[1] During: resolving callee type: Function(<numba.cuda.compiler.DeviceFunctionTemplate object at 0x7f3fd3601d90>)
[2] During: typing of call at <ipython-input-9-7e564dbb18c4> (17)
--%<-----------------------------------------------------------------

File "<ipython-input-9-7e564dbb18c4>", line 17

Failed at nopython (nopython frontend)
Internal error at <numba.typeinfer.CallConstraint object at 0x7f3fc80d5790>:
--%<-----------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typeinfer.py", line 128, in propagate
    constraint(typeinfer)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typeinfer.py", line 379, in __call__
    self.resolve(typeinfer, typevars, fnty)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/contextlib.py", line 35, in __exit__
    self.gen.throw(type, value, traceback)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/errors.py", line 249, in new_error_context
    six.reraise(type(newerr), newerr, sys.exc_info()[2])
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/errors.py", line 243, in new_error_context
    yield
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typeinfer.py", line 379, in __call__
    self.resolve(typeinfer, typevars, fnty)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typeinfer.py", line 392, in resolve
    sig = typeinfer.resolve_call(fnty, pos_args, kw_args)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typeinfer.py", line 1030, in resolve_call
    return self.context.resolve_function_type(fnty, pos_args, kw_args)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typing/context.py", line 202, in resolve_function_type
    return func.get_call_type(self, args, kws)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/types/functions.py", line 49, in get_call_type
    sig = temp.apply(args, kws)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/typing/templates.py", line 185, in apply
    sig = generic(args, kws)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.py", line 151, in generic
    return dft.compile(args)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.py", line 120, in compile
    inline=self.inline)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.py", line 37, in core
    return fn(*args, **kwargs)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/cuda/compiler.py", line 64, in compile_cuda
    locals={})
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 693, in compile_extra
    return pipeline.compile_extra(func)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 350, in compile_extra
    return self._compile_bytecode()
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 658, in _compile_bytecode
    return self._compile_core()
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 645, in _compile_core
    res = pm.run(self.status)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 236, in run
    raise patched_exception
InternalError: Caused By:
Traceback (most recent call last):
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 228, in run
    stage()
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 364, in stage_analyze_bytecode
    func_ir = translate_stage(self.func_id, self.bc)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/compiler.py", line 756, in translate_stage
    return interp.interpret(bytecode)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/interpreter.py", line 109, in interpret
    self._dispatch(inst, kws)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/interpreter.py", line 229, in _dispatch
    return fn(inst, **kws)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/interpreter.py", line 566, in op_LOAD_GLOBAL
    value = self.get_global_value(name)
  File "/usr/local/anaconda3/envs/py2gpu/lib/python2.7/site-packages/numba/interpreter.py", line 185, in get_global_value
    return utils.get_function_globals(self.func_id.func)[name]
AttributeError: 'CPUDispatcher' object has no attribute 'func_globals'

Failed at nopython (analyzing bytecode)
'CPUDispatcher' object has no attribute 'func_globals'
[1] During: resolving callee type: Function(<numba.cuda.compiler.DeviceFunctionTemplate object at 0x7f3fd3601d90>)
[2] During: typing of call at <ipython-input-9-7e564dbb18c4> (17)
--%<-----------------------------------------------------------------

File "<ipython-input-9-7e564dbb18c4>", line 17

My Current Environment

!nvidia-smi
Tue Sep 19 11:10:38 2017       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 384.66                 Driver Version: 384.66                    |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla P100-PCIE...  Off  | 00000000:0C:00.0 Off |                    0 |
| N/A   35C    P0    34W / 250W |    961MiB / 16276MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID  Type  Process name                               Usage      |
|=============================================================================|
|    0      7749    C   /usr/local/anaconda3/envs/py2gpu/bin/python    293MiB |
|    0     11190    C   /usr/local/anaconda3/envs/py2gpu/bin/python    329MiB |
|    0     33137    C   /usr/local/anaconda3/envs/py2gpu/bin/python    329MiB |
+-----------------------------------------------------------------------------+

import numba.cuda.api,numba.cuda.cudadrv.libs
 
numba.cuda.cudadrv.libs.test()
Finding cublas
	located at /usr/local/anaconda3/envs/py2gpu/lib/libcublas.so.8.0
	trying to open library...	ok
Finding cusparse
	located at /usr/local/anaconda3/envs/py2gpu/lib/libcusparse.so.8.0
	trying to open library...	ok
Finding cufft
	located at /usr/local/anaconda3/envs/py2gpu/lib/libcufft.so.8.0
	trying to open library...	ok
Finding curand
	located at /usr/local/anaconda3/envs/py2gpu/lib/libcurand.so.8.0
	trying to open library...	ok
Finding nvvm
	located at /usr/local/anaconda3/envs/py2gpu/lib/libnvvm.so.3.1.0
	trying to open library...	ok
	finding libdevice for compute_20...	ok
	finding libdevice for compute_30...	ok
	finding libdevice for compute_35...	ok
	finding libdevice for compute_50...	ok

numba.cuda.api.detect()
Found 1 CUDA devices
id 0    Tesla P100-PCIE-16GB                              [SUPPORTED]
                      compute capability: 6.0
                           pci device id: 0
                              pci bus id: 12
Summary:
	1/1 devices are supported

!conda list cudatoolkit
\# packages in environment at /usr/local/anaconda3/envs/py2gpu:
\#
cudatoolkit               8.0                           2    numba

!conda list numba
\# packages in environment at /usr/local/anaconda3/envs/py2gpu:
\#
numba                     0.30.1              np111py27_0  

!conda list accelerate
\# packages in environment at /usr/local/anaconda3/envs/py2gpu:
\#
accelerate                2.3.1               np111py27_0  
accelerate_cudalib        2.0                           0  

[harrism]

I can't repro. I have the same GPU. Consider updating Numba, removing accelerate (outdated), and installing pyculib. Here's my packages:

[mharris@hsw213 numba_examples]$ conda list cudatoolkit
# packages in environment at /home/mharris/anaconda2:
#
cudatoolkit               8.0                           1
[mharris@hsw213 numba_examples]$ conda list numba
# packages in environment at /home/mharris/anaconda2:
#
numba                     0.35.0              np111py27_0
[mharris@hsw213 numba_examples]$ conda list accelerate
# packages in environment at /home/mharris/anaconda2:
#

[mharris@hsw213 numba_examples]$ conda list pyculib
# packages in environment at /home/mharris/anaconda2:
#
pyculib                   1.0.2               np111py27_2
pyculib_sorting           1.0.0                         8

[gachet]

I run this version in kagle with no errors (except print) but the last image only present the background, the numba version is 0.55.3, the same result is obtained with colab, numba 0.56.3.

Regards and thank you