CMAKE'ing a CUDA static library of kernels issues

Accelerated Computing CUDA CUDA Programming and Performance
1

I am new to CMake. I have the following source files:
Screenshot 2023-09-29 at 09.26.06
I have the following two issues that I am bouncing between:
/usr/bin/ld: cannot find CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/cmake_device_link.o: No such file or directory
and
nvcc fatal : A single input file is required for a non-link phase when an outputfile is specified
And yes /cmake_device_link.o is not created though there is no errors that I can see as to why.

## [main]
if(NOT DEFINED ${CMAKE_BUILD_TYPE})
	set(CMAKE_BUILD_TYPE "Debug")
endif()

# Almost all CMake files should start with this
# You should always specify a range with the newest
# and oldest tested versions of CMake. This will ensure
# you pick up the best policies. CUDA started at v3.8.
cmake_minimum_required(VERSION 3.8...3.27)

message(STATUS ">>Static library CUDA kernels build started")

# set the project name
project(StaticLibraryMakeCUDAKernel
        VERSION 1.0
        DESCRIPTION "Static library holding CUDA kernels"
        LANGUAGES CXX CUDA)

# If you set any CMAKE_ variables, that can go here.
# (But usually don't do this, except maybe for C++ standard)
## [start set variables]
# Let's nicely support folders in IDEs
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
set(CMAKE_VERBOSE_MAKEFILE ON)
set(TARGET_NAME StaticLibOfFnsCUDAKernelcmake_d)
set(STATIC_LIB_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/StaticLibraryMakeCUDAKernel/src")
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -rdc=true")
string(APPEND CMAKE_CUDA_FLAGS " -c")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
    string(APPEND CMAKE_CXX_FLAGS " -G")
    string(APPEND CMAKE_CXX_FLAGS " -O0")
endif()
set(STATIC_LIB_FILES "${STATIC_LIB_INCLUDE_DIR}/library.cu")
#"${STATIC_LIB_INCLUDE_DIR}/cudakernels.cu" 
#"${STATIC_LIB_INCLUDE_DIR}/staticlibfnscudakernel.h")
## [end set variables]

# Find packages go here
## [find_package]
# Dynamic parallelism requires relocatable device code linking, in addition to compiling.
find_package(CUDA REQUIRED)
## [find_package]

## [core]
# Sets up global settings

# add other deeper CMakeList sub directories

# Create the static library
## [add_and_link]
add_library(${TARGET_NAME} SHARED ${STATIC_LIB_FILES})
# Request that particles be built with -std=c++11
# As this is a public compile feature anything that links to particles
# will also build with -std=c++11
target_compile_features(${TARGET_NAME} PUBLIC cxx_std_11)
# We need to explicitly state that we need all CUDA files in this
# library to be built with -dc as the member functions could be called by
# other libraries and executables
#set_property(TARGET ${TARGET_NAME} PROPERTY CUDA_SEPARABLE_COMPILATION ON)
set_target_properties(${TARGET_NAME} PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
# Enables both relocatable device code compiling and linking:
set_target_properties(${TARGET_NAME} PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
set_target_properties(${TARGET_NAME} PROPERTIES CUDA_ARCHITECTURES "75")
target_link_libraries(${TARGET_NAME} PUBLIC ${CMAKE_THREAD_LIBS_INIT}  ${CMAKE_DL_LIBS})
## [add_and_link]

## [core]

## [main]
message(STATUS "<<Static library CUDA kernels build finished")

The output from make is as follows when specifying more than one source file:

[ 16%] Building CUDA object StaticLibraryMakeCUDAKernel/CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/src/library.cu.o
[ 33%] Building CUDA object StaticLibraryMakeCUDAKernel/CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/src/cudakernels.cu.o
[ 50%] Linking CUDA device code CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/cmake_device_link.o
/usr/local/cuda-12.0/bin/nvcc -forward-unknown-to-host-compiler  -rdc=true -c -g --generate-code=arch=compute_75,code=[compute_75,sm_75] -Xcompiler=-fPIC -Wno-deprecated-gpu-targets -shared -dlink CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/src/library.cu.o CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/src/cudakernels.cu.o -o CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/cmake_device_link.o  -ldl -lcudadevrt -lcudart_static -lrt -lpthread 
**nvcc fatal   : A single input file is required for a non-link phase when an outputfile is specified**
make[2]: *** [StaticLibraryMakeCUDAKernel/CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/build.make:116: StaticLibraryMakeCUDAKernel/CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/cmake_device_link.o] Error 1
make[1]: *** [CMakeFiles/Makefile2:117: StaticLibraryMakeCUDAKernel/CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/all] Error 2
make: *** [Makefile:91: all] Error 2

or when a single .cu file is specified:

[ 20%] Building CUDA object StaticLibraryMakeCUDAKernel/CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/src/library.cu.o
[ 40%] Linking CUDA device code CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/cmake_device_link.o
/usr/local/cuda-12.0/bin/nvcc -forward-unknown-to-host-compiler  -rdc=true -c -g --generate-code=arch=compute_75,code=[compute_75,sm_75] -Xcompiler=-fPIC -Wno-deprecated-gpu-targets -shared -dlink CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/src/library.cu.o -o CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/cmake_device_link.o  -ldl -lcudadevrt -lcudart_static -lrt -lpthread 
[ 60%] Linking CUDA shared library libStaticLibOfFnsCUDAKernelcmake_d.so
/usr/bin/g++ -fPIC -shared -Wl,-soname,libStaticLibOfFnsCUDAKernelcmake_d.so -o libStaticLibOfFnsCUDAKernelcmake_d.so CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/src/library.cu.o CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/cmake_device_link.o  -ldl -lcudadevrt -lcudart_static -lrt -lpthread -ldl  -L"/usr/local/cuda-12.0/targets/x86_64-linux/lib/stubs" -L"/usr/local/cuda-12.0/targets/x86_64-linux/lib"
**/usr/bin/ld: cannot find CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/cmake_device_link.o: No such file or directory**
collect2: error: ld returned 1 exit status
make[2]: *** [StaticLibraryMakeCUDAKernel/CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/build.make:118: StaticLibraryMakeCUDAKernel/libStaticLibOfFnsCUDAKernelcmake_d.so] Error 1
make[1]: *** [CMakeFiles/Makefile2:117: StaticLibraryMakeCUDAKernel/CMakeFiles/StaticLibOfFnsCUDAKernelcmake_d.dir/all] Error 2
make: *** [Makefile:91: all] Error 2

If the first issue is not really a problem in that all the source files are found and compiled, then why is a cmake_device_link.o is not be produced?
Any clues as to how to fix this would be awesome.
Note: my other CMAKE statements may not be correct.

2

This issue may have turned into bug. A colleague has replicated the part where nvcc exits without reporting an issue or producing an output.

3

CMake isn’t an NVIDIA product. You’re welcome to ask about CMake here, but nobody has control over it at NVIDIA.

If you think there is a bug in nvcc you should plan to describe it and develop a full repro case (demonstrator) without using CMake in any way, i.e. demonstrate it using purely nvcc commands.

4

Taking a look at this command issued by CMake:

It is broken in at least a couple ways.

  1. As the error message indicates, it is explicitly asking for a compile phase due to the -c switch, but it is apparently attempting to run a link phase, based on the input files (.cu.o). Just as the error message says, when you ask for compilation only, and specify an output file, you cannot present multiple compilation targets. This is not a bug in nvcc.

Since this would be the command that generates cmake_device_link.o, it should be evident why that is not getting created. There is an error:

  1. If you are attempting to create a static library (just based on the title) the command is not the right one to use. For nvcc, the -shared switch creates a dynamic library (shared object), not a static library. This is not a bug in nvcc.

Sorry, I won’t be able to help you get CMake to do what you want. I’m not a CMake expert. But it is quite evident that the commands CMake is trying to present to nvcc are ill-formed, and that is not a bug in nvcc.

5

Thank you for looking at this. It has narrowed the scope of the issue.

6

To get around the issue with CMake in this regard I have used a mix and match solution with Make. This works.
Here is the Make file to create a static cuda kernel library:

build:
	$(CUDA_PATH)/nvcc -rdc=true -c -ccbin g++ -I$(INCLUDE_PATH) -I/usr/local/cuda/bin/../targets/x86_64-linux/include -m64 --std=c++11 -gencode arch=compute_75,code=sm_75 ../src/$(SRC_FILE) -o $(DST_FILE)_tmp.o 
	$(CUDA_PATH)/nvcc -dlink -o  libStaticLibOfFnsCUDAKernel.o  libStaticLibOfFnsCUDAKernel_tmp.o -lcudart
	ar cru  $(DST_FILE).a  $(DST_FILE).o $(DST_FILE)_tmp.o
	ranlib $(DST_FILE).a
7

The CMAKE script has now been fixed. This is the working file:

## [main]
if(NOT DEFINED ${CMAKE_BUILD_TYPE})
	set(CMAKE_BUILD_TYPE "Debug")
endif()

# Almost all CMake files should start with this
# You should always specify a range with the newest
# and oldest tested versions of CMake. This will ensure
# you pick up the best policies. CUDA started at v3.8.
cmake_minimum_required(VERSION 3.8...3.27)

message(STATUS ">>Static library CUDA kernels build started")

# set the project name
project(StaticLibraryMakeCUDAKernel
        VERSION 1.0
        DESCRIPTION "Static library holding CUDA kernels"
        LANGUAGES CXX CUDA)

# If you set any CMAKE_ variables, that can go here.
# (But usually don't do this, except maybe for C++ standard)
## [start set variables]
# Let's nicely support folders in IDEs
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
set(CMAKE_VERBOSE_MAKEFILE ON)
set(TARGET_NAME StaticLibOfFnsCUDAKernelcmake_d)
set(STATIC_LIB_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/StaticLibraryMakeCUDAKernel/src")
set(STATIC_LIB_FILES "${STATIC_LIB_INCLUDE_DIR}/library.cu")
## [end set variables]

# Find packages go here
## [find_package]
# The CUDA package has been deprecated and will cause errors when using CMake 3.27 or greater
#find_package(CUDA REQUIRED)
## [find_package]

## [core]
# Sets up global settings

# add other deeper CMakeList sub directories

# Create the static library
## [add_and_link]
add_library(${TARGET_NAME} SHARED ${STATIC_LIB_FILES})
# Request that particles be built with -std=c++11
# As this is a public compile feature anything that links to particles
# will also build with -std=c++11
target_compile_features(${TARGET_NAME} PUBLIC cxx_std_11)
# We need to explicitly state that we need all CUDA files in this
# library to be built with -dc as the member functions could be called by
# other libraries and executables
set_target_properties(${TARGET_NAME} PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
# Enables both relocatable device code compiling and linking:
set_target_properties(${TARGET_NAME} PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
set_target_properties(${TARGET_NAME} PROPERTIES CUDA_ARCHITECTURES "75")
# Redirect binaries to the project folder
#set_target_properties(${TARGET_NAME} PROPERTIES
#    ARCHIVE_OUTPUT_DIRECTORY "${PROJECT_HOME}"
#    LIBRARY_OUTPUT_DIRECTORY "${PROJECT_HOME}"
#    RUNTIME_OUTPUT_DIRECTORY "${PROJECT_HOME}"
target_link_libraries(${TARGET_NAME} PUBLIC ${CMAKE_THREAD_LIBS_INIT}  ${CMAKE_DL_LIBS})
## [add_and_link]

## [core]

## [main]
message(STATUS "<<Static library CUDA kernels build finished")

Notable changes are:

  • removal of the CUDA package as this causes issues
  • removal of the compiler and linker flags variable as handle by CMAKE itself
  • The -rdc=true is handled when you set CUDA_SEPARABLE_COMPILATION to ON.
    The addition of -c to CMAKE_CUDA_FLAGS breaks the device link phase since all CMAKE_<LANG>_FLAGS are propagated to respective link phases.
8

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.