I have a similar issue although nvidia-smi, nvcc --version, lsmod etc. looks correct to me (see below).
Issue(s)
$ ./deviceQuery
./deviceQuery Starting...
CUDA Device Query (Runtime API) version (CUDART static linking)
cudaGetDeviceCount returned 999
-> unknown error
Result = FAIL
$ ./mnistCUDNN
Executing: mnistCUDNN
cudnnGetVersion() : 8401 , CUDNN_VERSION from cudnn.h : 8401 (8.4.1)
Host compiler version : GCC 11.2.0
ERROR: cuda failure (unknown error) in error_util.h:91
Aborting...
Infos:
$ uname -r
5.15.0-40-generic
$ nvcc --version
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2022 NVIDIA Corporation
Built on Tue_May__3_18:49:52_PDT_2022
Cuda compilation tools, release 11.7, V11.7.64
Build cuda_11.7.r11.7/compiler.31294372_0
$ nvidia-smi
Fri Jul 8 08:07:13 2022
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 515.48.07 Driver Version: 515.48.07 CUDA Version: 11.7 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 NVIDIA GeForce ... On | 00000000:01:00.0 Off | N/A |
| N/A 35C P8 13W / N/A | 5MiB / 16384MiB | 0% Default |
| | | N/A |
+-------------------------------+----------------------+----------------------+
+-----------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=============================================================================|
| 0 N/A N/A 2093 G /usr/lib/xorg/Xorg 4MiB |
+-----------------------------------------------------------------------------+
$ apt list --installed|grep 'cuda\|nvidia'
WARNING: apt does not have a stable CLI interface. Use with caution in scripts.
cuda-11-7/unknown,now 11.7.0-1 amd64 [installed,automatic]
cuda-cccl-11-7/unknown,now 11.7.58-1 amd64 [installed,automatic]
cuda-command-line-tools-11-7/unknown,now 11.7.0-1 amd64 [installed,automatic]
cuda-compiler-11-7/unknown,now 11.7.0-1 amd64 [installed,automatic]
cuda-cudart-11-7/unknown,now 11.7.60-1 amd64 [installed,automatic]
cuda-cudart-dev-11-7/unknown,now 11.7.60-1 amd64 [installed,automatic]
cuda-cuobjdump-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-cupti-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-cupti-dev-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-cuxxfilt-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-demo-suite-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-documentation-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-driver-dev-11-7/unknown,now 11.7.60-1 amd64 [installed,automatic]
cuda-drivers-515/unknown,now 515.48.07-1 amd64 [installed,automatic]
cuda-drivers/unknown,now 515.48.07-1 amd64 [installed,automatic]
cuda-gdb-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-libraries-11-7/unknown,now 11.7.0-1 amd64 [installed,automatic]
cuda-libraries-dev-11-7/unknown,now 11.7.0-1 amd64 [installed,automatic]
cuda-memcheck-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-nsight-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-nsight-compute-11-7/unknown,now 11.7.0-1 amd64 [installed,automatic]
cuda-nsight-systems-11-7/unknown,now 11.7.0-1 amd64 [installed,automatic]
cuda-nvcc-11-7/unknown,now 11.7.64-1 amd64 [installed,automatic]
cuda-nvdisasm-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-nvml-dev-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-nvprof-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-nvprune-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-nvrtc-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-nvrtc-dev-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-nvtx-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-nvvp-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-repo-ubuntu2204-11-7-local/now 11.7.0-515.43.04-1 amd64 [installed,local]
cuda-runtime-11-7/unknown,now 11.7.0-1 amd64 [installed,automatic]
cuda-sanitizer-11-7/unknown,now 11.7.50-1 amd64 [installed,automatic]
cuda-toolkit-11-7-config-common/unknown,now 11.7.60-1 all [installed,automatic]
cuda-toolkit-11-7/unknown,now 11.7.0-1 amd64 [installed,automatic]
cuda-toolkit-11-config-common/unknown,now 11.7.60-1 all [installed,automatic]
cuda-toolkit-config-common/unknown,now 11.7.60-1 all [installed,automatic]
cuda-tools-11-7/unknown,now 11.7.0-1 amd64 [installed,automatic]
cuda-visual-tools-11-7/unknown,now 11.7.0-1 amd64 [installed,automatic]
cuda/unknown,now 11.7.0-1 amd64 [installed]
libcudnn8-dev/unknown,now 8.4.1.50-1+cuda11.6 amd64 [installed]
libcudnn8-samples/unknown,now 8.4.1.50-1+cuda11.6 amd64 [installed]
libcudnn8/unknown,now 8.4.1.50-1+cuda11.6 amd64 [installed]
libnvidia-cfg1-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
libnvidia-common-515/unknown,now 515.48.07-0ubuntu1 all [installed,automatic]
libnvidia-compute-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
libnvidia-compute-515/unknown,now 515.48.07-0ubuntu1 i386 [installed,automatic]
libnvidia-decode-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
libnvidia-decode-515/unknown,now 515.48.07-0ubuntu1 i386 [installed,automatic]
libnvidia-encode-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
libnvidia-encode-515/unknown,now 515.48.07-0ubuntu1 i386 [installed,automatic]
libnvidia-extra-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
libnvidia-fbc1-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
libnvidia-fbc1-515/unknown,now 515.48.07-0ubuntu1 i386 [installed,automatic]
libnvidia-gl-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
libnvidia-gl-515/unknown,now 515.48.07-0ubuntu1 i386 [installed,automatic]
linux-modules-nvidia-515-5.15.0-40-generic/jammy-updates,now 5.15.0-40.43+1 amd64 [installed,automatic]
linux-modules-nvidia-515-generic-hwe-22.04/jammy-updates,now 5.15.0-40.43+1 amd64 [installed]
linux-objects-nvidia-515-5.15.0-40-generic/jammy-updates,now 5.15.0-40.43+1 amd64 [installed,automatic]
linux-signatures-nvidia-5.15.0-40-generic/jammy-updates,now 5.15.0-40.43+1 amd64 [installed,automatic]
nvidia-compute-utils-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
nvidia-dkms-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
nvidia-driver-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
nvidia-kernel-common-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
nvidia-kernel-source-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
nvidia-modprobe/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
nvidia-prime/jammy,jammy,now 0.8.17.1 all [installed]
nvidia-settings/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
nvidia-utils-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
xserver-xorg-video-nvidia-515/unknown,now 515.48.07-0ubuntu1 amd64 [installed,automatic]
$ lsmod
Module Size Used by
uas 28672 0
usb_storage 77824 2 uas
rfcomm 81920 4
cmac 16384 3
algif_hash 16384 1
algif_skcipher 16384 1
af_alg 32768 6 algif_hash,algif_skcipher
snd_soc_skl_hda_dsp 28672 6
snd_soc_intel_hda_dsp_common 20480 1 snd_soc_skl_hda_dsp
snd_soc_hdac_hdmi 36864 1 snd_soc_skl_hda_dsp
hid_logitech_hidpp 49152 0
snd_hda_codec_realtek 151552 1
snd_hda_codec_generic 102400 1 snd_hda_codec_realtek
hid_logitech_dj 28672 0
bnep 28672 2
btusb 61440 0
btrtl 24576 1 btusb
btbcm 24576 1 btusb
uvcvideo 106496 0
btintel 40960 1 btusb
videobuf2_vmalloc 20480 1 uvcvideo
videobuf2_memops 20480 1 videobuf2_vmalloc
bluetooth 688128 33 btrtl,btintel,btbcm,bnep,btusb,rfcomm
videobuf2_v4l2 32768 1 uvcvideo
videobuf2_common 77824 4 videobuf2_vmalloc,videobuf2_v4l2,uvcvideo,videobuf2_memops
videodev 249856 3 videobuf2_v4l2,uvcvideo,videobuf2_common
ecdh_generic 16384 2 bluetooth
mc 65536 4 videodev,videobuf2_v4l2,uvcvideo,videobuf2_common
usbhid 65536 1 hid_logitech_dj
ecc 36864 1 ecdh_generic
intel_rapl_msr 20480 0
intel_rapl_common 36864 1 intel_rapl_msr
intel_tcc_cooling 16384 0
nvidia_drm 69632 2
nvidia_modeset 1142784 4 nvidia_drm
snd_soc_dmic 16384 1
snd_sof_pci_intel_tgl 16384 0
snd_sof_intel_hda_common 98304 1 snd_sof_pci_intel_tgl
soundwire_intel 40960 1 snd_sof_intel_hda_common
soundwire_generic_allocation 16384 1 soundwire_intel
soundwire_cadence 36864 1 soundwire_intel
snd_sof_intel_hda 20480 1 snd_sof_intel_hda_common
snd_sof_pci 20480 2 snd_sof_intel_hda_common,snd_sof_pci_intel_tgl
snd_sof_xtensa_dsp 16384 1 snd_sof_intel_hda_common
snd_sof 143360 2 snd_sof_pci,snd_sof_intel_hda_common
snd_soc_hdac_hda 24576 1 snd_sof_intel_hda_common
snd_hda_ext_core 32768 4 snd_sof_intel_hda_common,snd_soc_hdac_hdmi,snd_soc_hdac_hda,snd_sof_intel_hda
snd_soc_acpi_intel_match 61440 2 snd_sof_intel_hda_common,snd_sof_pci_intel_tgl
snd_soc_acpi 16384 2 snd_soc_acpi_intel_match,snd_sof_intel_hda_common
soundwire_bus 94208 3 soundwire_intel,soundwire_generic_allocation,soundwire_cadence
ledtrig_audio 16384 2 snd_hda_codec_generic,snd_sof
snd_soc_core 331776 7 soundwire_intel,snd_sof,snd_sof_intel_hda_common,snd_soc_hdac_hdmi,snd_soc_hdac_hda,snd_soc_dmic,snd_soc_skl_hda_dsp
snd_hda_codec_hdmi 73728 2
snd_compress 24576 1 snd_soc_core
iwlmvm 561152 0
ac97_bus 16384 1 snd_soc_core
snd_pcm_dmaengine 16384 1 snd_soc_core
snd_hda_intel 53248 1
snd_intel_dspcfg 28672 2 snd_hda_intel,snd_sof_intel_hda_common
snd_intel_sdw_acpi 20480 2 snd_sof_intel_hda_common,snd_intel_dspcfg
x86_pkg_temp_thermal 20480 0
snd_hda_codec 155648 7 snd_hda_codec_generic,snd_hda_codec_hdmi,snd_hda_intel,snd_hda_codec_realtek,snd_soc_intel_hda_dsp_common,snd_soc_hdac_hda,snd_soc_skl_hda_dsp
intel_powerclamp 20480 0
mac80211 1228800 1 iwlmvm
binfmt_misc 24576 1
snd_hda_core 110592 11 snd_hda_codec_generic,snd_hda_codec_hdmi,snd_hda_intel,snd_hda_ext_core,snd_hda_codec,snd_hda_codec_realtek,snd_soc_intel_hda_dsp_common,snd_sof_intel_hda_common,snd_soc_hdac_hdmi,snd_soc_hdac_hda,snd_sof_intel_hda
snd_hwdep 16384 1 snd_hda_codec
snd_pcm 139264 11 snd_hda_codec_hdmi,snd_hda_intel,snd_hda_codec,soundwire_intel,snd_sof,snd_sof_intel_hda_common,snd_soc_hdac_hdmi,snd_compress,snd_soc_core,snd_hda_core,snd_pcm_dmaengine
joydev 32768 0
snd_seq_midi 20480 0
nvidia 40804352 130 nvidia_modeset
libarc4 16384 1 mac80211
coretemp 24576 0
nls_iso8859_1 16384 2
snd_seq_midi_event 16384 1 snd_seq_midi
i915 3043328 18
kvm_intel 364544 0
kvm 1003520 1 kvm_intel
snd_rawmidi 49152 1 snd_seq_midi
iwlwifi 446464 1 iwlmvm
crct10dif_pclmul 16384 1
ghash_clmulni_intel 16384 0
aesni_intel 376832 4
snd_seq 73728 2 snd_seq_midi,snd_seq_midi_event
crypto_simd 16384 1 aesni_intel
ttm 86016 1 i915
cryptd 24576 3 crypto_simd,ghash_clmulni_intel
drm_kms_helper 307200 2 nvidia_drm,i915
mei_hdcp 24576 0
input_leds 16384 0
snd_seq_device 16384 3 snd_seq,snd_seq_midi,snd_rawmidi
hp_wmi 20480 0
sparse_keymap 16384 1 hp_wmi
intel_cstate 20480 0
cfg80211 958464 3 iwlmvm,iwlwifi,mac80211
platform_profile 16384 1 hp_wmi
serio_raw 20480 0
efi_pstore 16384 0
wmi_bmof 16384 0
snd_timer 40960 2 snd_seq,snd_pcm
cec 61440 2 drm_kms_helper,i915
mei_me 40960 1
ee1004 20480 0
nvidia_wmi_ec_backlight 16384 0
snd 102400 29 snd_hda_codec_generic,snd_seq,snd_seq_device,snd_hda_codec_hdmi,snd_hwdep,snd_hda_intel,snd_hda_codec,snd_hda_codec_realtek,snd_timer,snd_soc_hdac_hdmi,snd_compress,snd_soc_core,snd_pcm,snd_soc_skl_hda_dsp,snd_rawmidi
rc_core 65536 1 cec
hid_multitouch 28672 0
i2c_algo_bit 16384 1 i915
mei 135168 3 mei_hdcp,mei_me
fb_sys_fops 16384 1 drm_kms_helper
syscopyarea 16384 1 drm_kms_helper
ucsi_acpi 16384 0
soundcore 16384 1 snd
sysfillrect 20480 1 drm_kms_helper
typec_ucsi 45056 1 ucsi_acpi
sysimgblt 16384 1 drm_kms_helper
typec 65536 1 typec_ucsi
mac_hid 16384 0
acpi_pad 184320 0
acpi_tad 20480 0
sch_fq_codel 20480 2
ipmi_devintf 20480 0
ipmi_msghandler 122880 1 ipmi_devintf
msr 16384 4
parport_pc 49152 0
ppdev 24576 0
drm 606208 12 drm_kms_helper,nvidia,nvidia_drm,i915,ttm
lp 28672 0
parport 65536 3 parport_pc,lp,ppdev
ip_tables 32768 0
x_tables 53248 1 ip_tables
autofs4 49152 2
hid_generic 16384 0
nvme 45056 2
nvme_core 126976 3 nvme
crc32_pclmul 16384 0
sdhci_pci 65536 0
i2c_i801 36864 0
i2c_smbus 20480 1 i2c_i801
cqhci 36864 1 sdhci_pci
i2c_hid_acpi 16384 0
r8169 98304 0
sdhci 81920 1 sdhci_pci
intel_lpss_pci 24576 0
i2c_hid 32768 1 i2c_hid_acpi
thunderbolt 307200 0
intel_lpss 16384 1 intel_lpss_pci
xhci_pci 24576 0
realtek 32768 1
idma64 20480 0
vmd 24576 0
xhci_pci_renesas 20480 1 xhci_pci
hid 147456 6 i2c_hid,usbhid,hid_multitouch,hid_generic,hid_logitech_dj,hid_logitech_hidpp
wmi 32768 3 hp_wmi,nvidia_wmi_ec_backlight,wmi_bmof
video 53248 1 i915
pinctrl_tigerlake 32768 0