Thanks to J. Bigler I have managed to make a cmake file that works at least on 64bit windows and linux, when using VS2008.
I have a few files that need to be compiled with -maxrregcount 85, so you will see below how to have some kernels compiled with different nvcc flags.
cmake_minimum_required(VERSION 2.8)
set_property(GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS 1)
FIND_PACKAGE(Matlab)
INCLUDE_DIRECTORIES(${MATLAB_INCLUDE_DIR})
FIND_PACKAGE(CUDA)
set(NO_LIMIT cuda_mex_headers.cu helpers.cu kernel1.cu)
set(LIMIT_85 kernel2.cu kernel3.cu)
set(MEX_FILE cuda_mex_kernel.cu )
set(CUDA_NVCC_FLAGS -arch sm_13;--ptxas-options=-v)
CUDA_COMPILE( generated_files0 ${NO_LIMIT} SHARED)
CUDA_COMPILE( generated_files1 ${LIMIT_85} SHARED OPTIONS -maxrregcount 85)
CUDA_COMPILE( cuda_mex_kernel_generated ${MEX_FILE} SHARED)
ADD_LIBRARY(cuda_mex_kernel SHARED ${cuda_mex_kernel_generated} ${MEX_FILE}
${generated_files0}
${NO_LIMIT}
${generated_files1}
${LIMIT_85}
)
TARGET_LINK_LIBRARIES(cuda_mex_kernel ${MATLAB_LIBRARIES} ${CUDA_LIBRARIES})
SET_TARGET_PROPERTIES(cuda_mex_kernel PROPERTIES PREFIX "" LINKER_LANGUAGE CXX)
if(WIN32)
if (CMAKE_CL_64)
MESSAGE("Win 64")
SET_TARGET_PROPERTIES(rtt_kernel PROPERTIES
SUFFIX .mexw64
LINK_FLAGS /export:mexFunction
)
else()
MESSAGE("Win 32")
SET_TARGET_PROPERTIES(rtt_kernel PROPERTIES
SUFFIX .mexw32
LINK_FLAGS /export:mexFunction
)
endif()
else(UNIX)
if (CMAKE_SIZEOF_VOID_P MATCHES "8")
MESSAGE("Linux 64")
SET_TARGET_PROPERTIES(rtt_kernel PROPERTIES SUFFIX .mexa64)
else()
MESSAGE("Linux 32")
SET_TARGET_PROPERTIES(rtt_kernel PROPERTIES SUFFIX .mexglx)
endif (CMAKE_SIZEOF_VOID_P MATCHES "8")
endif()