CUDA error: CUBLAS_STATUS_NOT_INITIALIZED when calling `cublasCreate(handle).
This need libcublas-dev package and eventually depends on ‘wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/7fa2af80.pub’ which is depreciated. i tried public key installation and tried to add deb package in source list as well but it did not work out.
the output from my console is
RL device: cuda:0
agent_dof_index: [[0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]]
hand_actuated_dof_index_dict: [[-1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 6, 7, 8, 10, 11, 12, 14, 15, 16, 17, 19, 20, 21, 22, 23], [-1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 6, 7, 8, 10, 11, 12, 14, 15, 16, 17, 19, 20, 21, 22, 23]]
/home/rupam/miniconda3/envs/rlgpu/lib/python3.7/site-packages/gym/spaces/box.py:127: UserWarning: WARN: Box bound precision lowered by casting to float32
logger.warn(f"Box bound precision lowered by casting to {self.dtype}")
Actor(
(base): MLPBase(
(feature_norm): LayerNorm((226,), eps=1e-05, elementwise_affine=True)
(mlp): MLPLayer(
(fc1): Sequential(
(0): Linear(in_features=226, out_features=512, bias=True)
(1): ELU(alpha=1.0)
(2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
(fc2): ModuleList(
(0): Sequential(
(0): Linear(in_features=512, out_features=512, bias=True)
(1): ELU(alpha=1.0)
(2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
(1): Sequential(
(0): Linear(in_features=512, out_features=512, bias=True)
(1): ELU(alpha=1.0)
(2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
)
)
)
(act): ACTLayer(
(action_out): DiagGaussian(
(fc_mean): Linear(in_features=512, out_features=26, bias=True)
)
)
)
Critic(
(base): MLPBase(
(feature_norm): LayerNorm((425,), eps=1e-05, elementwise_affine=True)
(mlp): MLPLayer(
(fc1): Sequential(
(0): Linear(in_features=425, out_features=512, bias=True)
(1): ELU(alpha=1.0)
(2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
(fc2): ModuleList(
(0): Sequential(
(0): Linear(in_features=512, out_features=512, bias=True)
(1): ELU(alpha=1.0)
(2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
(1): Sequential(
(0): Linear(in_features=512, out_features=512, bias=True)
(1): ELU(alpha=1.0)
(2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
)
)
)
(v_out): Linear(in_features=512, out_features=1, bias=True)
)
Actor(
(base): MLPBase(
(feature_norm): LayerNorm((226,), eps=1e-05, elementwise_affine=True)
(mlp): MLPLayer(
(fc1): Sequential(
(0): Linear(in_features=226, out_features=512, bias=True)
(1): ELU(alpha=1.0)
(2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
(fc2): ModuleList(
(0): Sequential(
(0): Linear(in_features=512, out_features=512, bias=True)
(1): ELU(alpha=1.0)
(2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
(1): Sequential(
(0): Linear(in_features=512, out_features=512, bias=True)
(1): ELU(alpha=1.0)
(2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
)
)
)
(act): ACTLayer(
(action_out): DiagGaussian(
(fc_mean): Linear(in_features=512, out_features=26, bias=True)
)
)
)
Critic(
(base): MLPBase(
(feature_norm): LayerNorm((425,), eps=1e-05, elementwise_affine=True)
(mlp): MLPLayer(
(fc1): Sequential(
(0): Linear(in_features=425, out_features=512, bias=True)
(1): ELU(alpha=1.0)
(2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
(fc2): ModuleList(
(0): Sequential(
(0): Linear(in_features=512, out_features=512, bias=True)
(1): ELU(alpha=1.0)
(2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
(1): Sequential(
(0): Linear(in_features=512, out_features=512, bias=True)
(1): ELU(alpha=1.0)
(2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
)
)
)
(v_out): Linear(in_features=512, out_features=1, bias=True)
)
Traceback (most recent call last):
File “train.py”, line 95, in
train()
File “train.py”, line 36, in train
runner.run()
File “/home/rupam/rl_work/DexterousHands/bidexhands/algorithms/marl/runner.py”, line 129, in run
values, actions, action_log_probs, rnn_states, rnn_states_critic = self.collect(step)
File “/home/rupam/miniconda3/envs/rlgpu/lib/python3.7/site-packages/torch/autograd/grad_mode.py”, line 27, in decorate_context
return func(*args, **kwargs)
File “/home/rupam/rl_work/DexterousHands/bidexhands/algorithms/marl/runner.py”, line 211, in collect
self.buffer[agent_id].masks[step])
File “/home/rupam/rl_work/DexterousHands/bidexhands/algorithms/marl/happo_policy.py”, line 80, in get_actions
deterministic)
File “/home/rupam/miniconda3/envs/rlgpu/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 889, in _call_impl
result = self.forward(*input, **kwargs)
File “/home/rupam/rl_work/DexterousHands/bidexhands/algorithms/marl/actor_critic.py”, line 62, in forward
actor_features = self.base(obs)
File “/home/rupam/miniconda3/envs/rlgpu/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 889, in _call_impl
result = self.forward(*input, **kwargs)
File “/home/rupam/rl_work/DexterousHands/bidexhands/algorithms/utils/mlp.py”, line 63, in forward
x = self.mlp(x)
File “/home/rupam/miniconda3/envs/rlgpu/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 889, in _call_impl
result = self.forward(*input, **kwargs)
File “/home/rupam/rl_work/DexterousHands/bidexhands/algorithms/utils/mlp.py”, line 32, in forward
x = self.fc1(x)
File “/home/rupam/miniconda3/envs/rlgpu/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 889, in _call_impl
result = self.forward(*input, **kwargs)
File “/home/rupam/miniconda3/envs/rlgpu/lib/python3.7/site-packages/torch/nn/modules/container.py”, line 119, in forward
input = module(input)
File “/home/rupam/miniconda3/envs/rlgpu/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 889, in _call_impl
result = self.forward(*input, **kwargs)
File “/home/rupam/miniconda3/envs/rlgpu/lib/python3.7/site-packages/torch/nn/modules/linear.py”, line 94, in forward
return F.linear(input, self.weight, self.bias)
File “/home/rupam/miniconda3/envs/rlgpu/lib/python3.7/site-packages/torch/nn/functional.py”, line 1753, in linear
return torch._C._nn.linear(input, weight, bias)
RuntimeError: CUDA error: CUBLAS_STATUS_NOT_INITIALIZED when calling cublasCreate(handle)
one more important point i am using NVIDIA-SMI 515.105.01 Driver Version: 515.105.01 CUDA Version: 11.7 for my simulation.
Any help would be great full.