The pytorch DEBUG build crashes cicc
nvcc error : 'cicc' died due to signal 11 (Invalid memory reference)
nvcc error : 'cicc' core dumped
Here’s a sample (generated) command that causes a crash
/opt/cuda/bin/nvcc -forward-unknown-to-host-compiler -DAT_PER_OPERATOR_HEADERS -DHAVE_MALLOC_USABLE_SIZE=1 -DHAVE_MMAP=1 -DHAVE_SHM_OPEN=1 -DHAVE_SHM_UNLINK=1 -DIDEEP_USE_MKL -DMINIZ_DISABLE_ZIP_READER_CRC32_CHECKS -DONNXIFI_ENABLE_EXT=1 -DONNX_ML=1 -DONNX_NAMESPACE=onnx_torch -DTORCH_CUDA_BUILD_MAIN_LIB -DUSE_C10D_GLOO -DUSE_C10D_MPI -DUSE_C10D_NCCL -DUSE_CUDA -DUSE_DISTRIBUTED -DUSE_EXPERIMENTAL_CUDNN_V8_API -DUSE_EXTERNAL_MZCRC -DUSE_FLASH_ATTENTION -DUSE_NCCL -DUSE_RPC -DUSE_TENSORPIPE -D_FILE_OFFSET_BITS=64 -Dtorch_cuda_EXPORTS -I/home/ognian/code/pytorch/cmake-build-debug/aten/src -I/home/ognian/code/pytorch/aten/src -I/home/ognian/code/pytorch/cmake-build-debug -I/home/ognian/code/pytorch -I/home/ognian/code/pytorch/third_party/onnx -I/home/ognian/code/pytorch/cmake-build-debug/third_party/onnx -I/home/ognian/code/pytorch/third_party/foxi -I/home/ognian/code/pytorch/cmake-build-debug/third_party/foxi -I/home/ognian/code/pytorch/aten/src/THC -I/home/ognian/code/pytorch/aten/src/ATen/cuda -I/home/ognian/code/pytorch/aten/src/ATen/../../../third_party/cutlass/include -I/home/ognian/code/pytorch/cmake-build-debug/caffe2/aten/src -I/home/ognian/code/pytorch/aten/src/ATen/.. -I/home/ognian/code/pytorch/cmake-build-debug/nccl/include -I/home/ognian/code/pytorch/c10/cuda/../.. -I/home/ognian/code/pytorch/c10/.. -I/home/ognian/code/pytorch/third_party/tensorpipe -I/home/ognian/code/pytorch/cmake-build-debug/third_party/tensorpipe -I/home/ognian/code/pytorch/third_party/tensorpipe/third_party/libnop/include -I/home/ognian/code/pytorch/torch/csrc/api -I/home/ognian/code/pytorch/torch/csrc/api/include -isystem /home/ognian/code/pytorch/cmake-build-debug/third_party/gloo -isystem /home/ognian/code/pytorch/cmake/../third_party/gloo -isystem /home/ognian/code/pytorch/third_party/protobuf/src -isystem /opt/intel/oneapi/mkl/latest/include -isystem /home/ognian/code/pytorch/third_party/gemmlowp -isystem /home/ognian/code/pytorch/third_party/neon2sse -isystem /home/ognian/code/pytorch/third_party/XNNPACK/include -isystem /home/ognian/code/pytorch/third_party/ittapi/include -isystem /home/ognian/code/pytorch/cmake/../third_party/eigen -isystem /opt/cuda/include -isystem /home/ognian/code/pytorch/third_party/ideep/include -isystem /home/ognian/code/pytorch/third_party/ideep/mkl-dnn/include -isystem /home/ognian/code/pytorch/cmake/../third_party/cudnn_frontend/include -Xfatbin -compress-all -DONNX_NAMESPACE=onnx_torch -gencode arch=compute_86,code=sm_86 -Xcudafe --diag_suppress=cc_clobber_ignored,--diag_suppress=set_but_not_used,--diag_suppress=field_without_dll_interface,--diag_suppress=base_class_has_different_dll_interface,--diag_suppress=dll_interface_conflict_none_assumed,--diag_suppress=dll_interface_conflict_dllexport_assumed,--diag_suppress=bad_friend_decl --expt-relaxed-constexpr --expt-extended-lambda -Wno-deprecated-gpu-targets --expt-extended-lambda -DCUB_WRAPPED_NAMESPACE=at_cuda_detail -DCUDA_HAS_FP16=1 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -g -g -lineinfo --source-in-ptx -std=c++17 -Xcompiler=-fPIC -DCAFFE2_USE_GLOO -DTH_HAVE_THREAD -Xcompiler=-Wall,-Wextra,-Wno-unused-parameter,-Wno-unused-function,-Wno-unused-result,-Wno-missing-field-initializers,-Wno-unknown-pragmas,-Wno-type-limits,-Wno-array-bounds,-Wno-unknown-pragmas,-Wno-strict-overflow,-Wno-strict-aliasing,-Wno-missing-braces,-Wno-maybe-uninitialized -MD -MT caffe2/CMakeFiles/torch_cuda.dir/__/aten/src/ATen/native/cuda/DistributionCauchyKernel.cu.o -MF caffe2/CMakeFiles/torch_cuda.dir/__/aten/src/ATen/native/cuda/DistributionCauchyKernel.cu.o.d -x cu -c /home/ognian/code/pytorch/aten/src/ATen/native/cuda/DistributionCauchyKernel.cu -o caffe2/CMakeFiles/torch_cuda.dir/__/aten/src/ATen/native/cuda/DistributionCauchyKernel.cu.o
Removing the --source-in-ptx flag prevents the crash.
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2023 NVIDIA Corporation
Built on Mon_Apr__3_17:16:06_PDT_2023
Cuda compilation tools, release 12.1, V12.1.105
Build cuda_12.1.r12.1/compiler.32688072_0