Start of rest of err w/ numba 0.56.4:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <numba.cuda.cudadrv.driver.Driver object at 0x7f819d257bb0>, fname = 'cuLaunchCooperativeKernel', retcode = 719
def _check_ctypes_error(self, fname, retcode):
if retcode != enums.CUDA_SUCCESS:
errname = ERROR_MAP.get(retcode, "UNKNOWN_CUDA_ERROR")
msg = "Call to %s results in %s" % (fname, errname)
_logger.error(msg)
if retcode == enums.CUDA_ERROR_NOT_INITIALIZED:
self._detect_fork()
> raise CudaAPIError(retcode, msg)
E numba.cuda.cudadrv.driver.CudaAPIError: [719] Call to cuLaunchCooperativeKernel results in CUDA_ERROR_LAUNCH_FAILED
../../../../.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py:388: CudaAPIError
================================================== slowest 5 durations ===================================================
349.08s call tests/context/cuda/adapt/cluster/accumulate/test_accumulate.py::test_ssc_accumulate_stats[2_levels]
293.38s call tests/context/cuda/adapt/cluster/accumulate/test_accumulate.py::test_ssc_calculate_cluster_entropy[cross_2]
2.85s call tests/context/cuda/adapt/cluster/accumulate/test_accumulate.py::test_ssc_accumulate_stats[cross_2]
0.20s call tests/context/cuda/adapt/cluster/accumulate/test_accumulate.py::test_ssc_calculate_cluster_entropy[cross_2_x_samples]
0.20s call tests/context/cuda/adapt/cluster/accumulate/test_accumulate.py::test_ssc_calculate_cluster_entropy[cluster_2_levels]
================================================ short test summary info =================================================
FAILED tests/context/cuda/adapt/cluster/accumulate/test_accumulate.py::test_ssc_accumulate_stats[cross_2] - numba.cuda.cudadrv.driver.CudaAPIError: [719] Call to cuLaunchCooperativeKernel results in CUDA_ERROR_LAUNCH_FAILED
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! stopping after 1 failures !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
======================================== 1 failed, 4 passed in 648.22s (0:10:48) =========================================
Traceback (most recent call last):
File "/usr/local/anaconda3/lib/python3.9/weakref.py", line 667, in _exitfunc
f()
File "/usr/local/anaconda3/lib/python3.9/weakref.py", line 591, in __call__
return info.func(*info.args, **(info.kwargs or {}))
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 2162, in deref
mem.free()
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 1951, in free
self._finalizer()
File "/usr/local/anaconda3/lib/python3.9/weakref.py", line 591, in __call__
return info.func(*info.args, **(info.kwargs or {}))
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 1603, in core
deallocations.add_item(driver.cuMemFree, ptr, size)
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 1169, in add_item
self.clear()
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 1180, in clear
dtor(handle)
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 320, in safe_cuda_api_call
self._check_ctypes_error(fname, retcode)
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 388, in _check_ctypes_error
raise CudaAPIError(retcode, msg)
numba.cuda.cudadrv.driver.CudaAPIError: [719] Call to cuMemFree results in CUDA_ERROR_LAUNCH_FAILED
Traceback (most recent call last):
File "/usr/local/anaconda3/lib/python3.9/weakref.py", line 667, in _exitfunc
f()
File "/usr/local/anaconda3/lib/python3.9/weakref.py", line 591, in __call__
return info.func(*info.args, **(info.kwargs or {}))
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 2162, in deref
mem.free()
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 1951, in free
self._finalizer()
File "/usr/local/anaconda3/lib/python3.9/weakref.py", line 591, in __call__
return info.func(*info.args, **(info.kwargs or {}))
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 1603, in core
deallocations.add_item(driver.cuMemFree, ptr, size)
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 1169, in add_item
self.clear()
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 1180, in clear
dtor(handle)
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 320, in safe_cuda_api_call
self._check_ctypes_error(fname, retcode)
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 388, in _check_ctypes_error
raise CudaAPIError(retcode, msg)
numba.cuda.cudadrv.driver.CudaAPIError: [719] Call to cuMemFree results in CUDA_ERROR_LAUNCH_FAILED
Traceback (most recent call last):
File "/usr/local/anaconda3/lib/python3.9/weakref.py", line 667, in _exitfunc
f()
File "/usr/local/anaconda3/lib/python3.9/weakref.py", line 591, in __call__
return info.func(*info.args, **(info.kwargs or {}))
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 2162, in deref
mem.free()
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 1951, in free
self._finalizer()
File "/usr/local/anaconda3/lib/python3.9/weakref.py", line 591, in __call__
return info.func(*info.args, **(info.kwargs or {}))
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 1603, in core
deallocations.add_item(driver.cuMemFree, ptr, size)
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 1169, in add_item
self.clear()
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 1180, in clear
dtor(handle)
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 320, in safe_cuda_api_call
self._check_ctypes_error(fname, retcode)
File "/home/shauncutts/.cache/pypoetry/virtualenvs/timbuktu-6r8-oA4b-py3.9/lib/python3.9/site-packages/numba/cuda/cudadrv/driver.py", line 388, in _check_ctypes_error
raise CudaAPIError(retcode, msg)
numba.cuda.cudadrv.driver.CudaAPIError: [719] Call to cuMemFree results in CUDA_ERROR_LAUNCH_FAILED