During running the follwing example with sanitizer:
https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/examples.html#example-1-single-process-single-thread-multiple-devices
I am facing the next error:
[Yehonatans@nlirSrv1 build2]$ /usr/local/cuda-11.7/bin/compute-sanitizer --tool memcheck ./utils/hpc/unit_testing/NcclToySingleThreadMultiGpu
========= COMPUTE-SANITIZER
========= Program hit CUDA_ERROR_NOT_FOUND (error 500) due to "named symbol not found" on CUDA API call to cuGetProcAddress.
========= Saved host backtrace up to driver entry point at error
========= Host Frame: [0x2d1e3a]
========= in /lib64/libcuda.so.1
========= Host Frame:misc/cudawrap.cc:149:cudaLibraryInit() [0x5ae6f]
========= in /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/11.7/nccl/lib/libnccl.so.2
========= Host Frame:/dvs/p4/build/sw/gpgpu/nccl/gitfusion/stable/src/init.cc:1117:ncclCommInitAll [0x3b392]
========= in /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/11.7/nccl/lib/libnccl.so.2
========= Host Frame:/home/Yehonatans/work//src/utils/hpc/unit_testing/NcclToySingleThreadMultiGpu.cu:53:main [0x499e]
========= in /home/Yehonatans/work//src/build2/./utils/hpc/unit_testing/NcclToySingleThreadMultiGpu
========= Host Frame:__libc_start_main [0x3acf3]
========= in /lib64/libc.so.6
========= Host Frame:_start [0x45de]
========= in /home/Yehonatans/work//src/build2/./utils/hpc/unit_testing/NcclToySingleThreadMultiGpu
========= Program hit cudaErrorPeerAccessAlreadyEnabled (error 704) due to "peer access is already enabled" on CUDA API call to cudaGetLastError.
========= Saved host backtrace up to driver entry point at error
========= Host Frame: [0x424d26]
========= in /lib64/libcuda.so.1
========= Host Frame:cudaGetLastError [0x105204]
========= in /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/11.7/nccl/lib/libnccl.so.2
========= Host Frame:transport/p2p.cc:304:p2pSendConnect(ncclComm*, ncclConnect*, int, int, ncclConnector*) [0x63ab5]
========= in /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/11.7/nccl/lib/libnccl.so.2
========= Host Frame:/dvs/p4/build/sw/gpgpu/nccl/gitfusion/stable/src/transport.cc:124:ncclTransportP2pSetup(ncclComm*, ncclTopoGraph*, int, int*) [0x4842f]
========= in /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/11.7/nccl/lib/libnccl.so.2
========= Host Frame:/dvs/p4/build/sw/gpgpu/nccl/gitfusion/stable/src/init.cc:773:initTransportsRank(ncclComm*, ncclUniqueId*) [0x38a52]
========= in /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/11.7/nccl/lib/libnccl.so.2
========= Host Frame:/dvs/p4/build/sw/gpgpu/nccl/gitfusion/stable/src/init.cc:1045:ncclCommInitRankFunc(ncclAsyncJob*) [0x3a7f3]
========= in /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/11.7/nccl/lib/libnccl.so.2
========= Host Frame:/dvs/p4/build/sw/gpgpu/nccl/gitfusion/stable/src/group.cc:42:ncclAsyncJobMain(void*) [0x514fb]
========= in /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/11.7/nccl/lib/libnccl.so.2
========= Host Frame:start_thread [0x81cf]
========= in /lib64/libpthread.so.0
========= Host Frame:clone [0x39dd3]
========= in /lib64/libc.so.6
=========
I am compile the project using cmake
set(CMAKE_CUDA_ARCHITECTURES 86 80) set(CMAKE_CUDA_COMPILER "$ENV{NVHPC_PATH}//cuda/bin/nvcc") set(CMAKE_CUDA_COMPILER_FORCED TRUE) project(Test C CXX CUDA) set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "$ENV{NVHPC_PATH}/cmake" ) find_package(NVHPC REQUIRED ) ADD_EXECUTABLE(NcclToySingleThreadMultiGpu NcclToySingleThreadMultiGpu.cu) target_link_directories(NcclToySingleThreadMultiGpu PUBLIC ${NVHPC_CUDA_LIBRARY_DIR}) TARGET_LINK_LIBRARIES(NcclToySingleThreadMultiGpu NVHPC::NCCL NVHPC::CUDA muparser pthread)
My setup is:
GPU rtx3090 + nvlink
NVHPC 22.7
OS Rocky 8
what am I doing wrong and how can I over come the issue?