Using TensorRT3.0 to convert tensorflow model to create TensorRT engine

Hi, I am using TX1. When I try to convert tensorflow model to TensorRT3.0 engine on host machine following the TensorRT 3.0 guide, I face a error. Concretely, when I run

engine = trt.utils.uff_to_trt_engine(...)

I got the below error message:

*** Error in `python3': free(): invalid next size (fast): 0x00000000067df700 ***
======= Backtrace: =========
/lib/x86_64-linux-gnu/libc.so.6(+0x777e5)[0x7fae5bf597e5]
/lib/x86_64-linux-gnu/libc.so.6(+0x8037a)[0x7fae5bf6237a]
/lib/x86_64-linux-gnu/libc.so.6(cfree+0x4c)[0x7fae5bf6653c]
/usr/local/cuda-8.0/lib64/libcudnn.so.6(cudnnDestroyConvolutionDescriptor+0x9)[0x7fae25e9ac69]
/usr/lib/x86_64-linux-gnu/libnvinfer.so.4(_ZN8nvinfer15cudnn21CudnnConvolutionLayer15getValidTacticsERKNS0_18EngineBuildContextE+0x1cc)[0x7fade2b2a63c]
/usr/lib/x86_64-linux-gnu/libnvinfer.so.4(_ZN8nvinfer15cudnn33selectFastestLayerAndDeleteOthersERNS0_18EngineBuildContextERKSt6vectorIPNS0_5LayerESaIS5_EE+0x7a9)[0x7fade2add6a9]
/usr/lib/x86_64-linux-gnu/libnvinfer.so.4(_ZN8nvinfer17builder16buildSingleLayerERNS_5cudnn18EngineBuildContextERNS0_4NodeERKSt13unordered_mapISsSt10unique_ptrINS1_6RegionESt14default_deleteIS8_EESt4hashISsESt8equal_toISsESaISt4pairIKSsSB_EEERNS_14CpuMemoryGroupEPS6_ISsSt6vectorIfSaIfEESD_SF_SaISG_ISH_SR_EEEb+0x509)[0x7fade2ac1119]
/usr/lib/x86_64-linux-gnu/libnvinfer.so.4(_ZN8nvinfer17builder18EngineTacticSupply13getBestTacticERNS0_4NodeERKNS_5query5PortsINS_13RegionFormatLEEEb+0x2e9)[0x7fade2ac3499]
/usr/lib/x86_64-linux-gnu/libnvinfer.so.4(+0x398d04)[0x7fade2b02d04]
/usr/lib/x86_64-linux-gnu/libnvinfer.so.4(_ZN8nvinfer17builder23chooseFormatsAndTacticsERNS0_5GraphERNS0_12TacticSupplyEPSt13unordered_mapISsSt6vectorIfSaIfEESt4hashISsESt8equal_toISsESaISt4pairIKSsS8_EEE+0xba0)[0x7fade2b05740]
/usr/lib/x86_64-linux-gnu/libnvinfer.so.4(_ZN8nvinfer17builder19makeEngineFromGraphERKNS_21CudaEngineBuildConfigERKNS_5cudnn15HardwareContextERNS0_5GraphEPSt13unordered_mapISsSt6vectorIfSaIfEESt4hashISsESt8equal_toISsESaISt4pairIKSsSD_EEEi+0x26e)[0x7fade2ac470e]
/usr/lib/x86_64-linux-gnu/libnvinfer.so.4(_ZN8nvinfer17builder11buildEngineERNS_21CudaEngineBuildConfigERKNS_5cudnn15HardwareContextERKNS_7NetworkE+0x5a3)[0x7fade2ac8183]
/usr/lib/x86_64-linux-gnu/libnvinfer.so.4(_ZN8nvinfer17Builder15buildCudaEngineERNS_18INetworkDefinitionE+0x11)[0x7fade2aa4dd1]
/usr/lib/python3.5/dist-packages/tensorrt/infer/_nv_infer_bindings.cpython-35m-x86_64-linux-gnu.so(+0x20161)[0x7fadf8c0a161]
python3(PyCFunction_Call+0x4f)[0x4e9b7f]
python3(PyEval_EvalFrameEx+0x614)[0x5372f4]
python3[0x540199]
python3(PyEval_EvalFrameEx+0x50b2)[0x53bd92]
python3[0x540199]
python3(PyEval_EvalCode+0x1f)[0x540e4f]
python3[0x60c272]
python3(PyRun_FileExFlags+0x9a)[0x60e71a]
python3(PyRun_SimpleFileExFlags+0x1bc)[0x60ef0c]
python3(Py_Main+0x456)[0x63fb26]
python3(main+0xe1)[0x4cfeb1]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf0)[0x7fae5bf02830]
python3(_start+0x29)[0x5d6049]
======= Memory map: ========
00400000-007a9000 r-xp 00000000 08:13 3407886                            /usr/bin/python3.5
009a9000-009ab000 r--p 003a9000 08:13 3407886                            /usr/bin/python3.5
009ab000-00a42000 rw-p 003ab000 08:13 3407886                            /usr/bin/python3.5
00a42000-00a73000 rw-p 00000000 00:00 0 
010a2000-0fbb3000 rw-p 00000000 00:00 0                                  [heap]
10000000-10001000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10001000-10002000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10002000-10003000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10003000-10004000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10004000-10005000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10005000-10006000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10006000-10007000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10007000-10008000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10008000-10009000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10009000-1000a000 rw-s 00000000 00:06 16414                              /dev/nvidia0
1000a000-1000b000 rw-s 00000000 00:06 16414                              /dev/nvidia0
1000b000-1000c000 rw-s 00000000 00:06 16414                              /dev/nvidia0
1000c000-1000d000 rw-s 00000000 00:06 16414                              /dev/nvidia0
1000d000-1000e000 rw-s 00000000 00:06 16414                              /dev/nvidia0
1000e000-1000f000 rw-s 00000000 00:06 16414                              /dev/nvidia0
1000f000-10010000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10010000-10011000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10011000-10012000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10012000-10013000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10013000-10014000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10014000-10015000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10015000-10016000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10016000-10017000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10017000-10018000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10018000-10019000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10019000-1001a000 rw-s 00000000 00:06 16414                              /dev/nvidia0
1001a000-1001b000 rw-s 00000000 00:06 16414                              /dev/nvidia0
1001b000-1001c000 rw-s 00000000 00:06 16414                              /dev/nvidia0
1001c000-1001d000 rw-s 00000000 00:06 16414                              /dev/nvidia0
1001d000-1001e000 rw-s 00000000 00:06 16414                              /dev/nvidia0
1001e000-1001f000 rw-s 00000000 00:06 16414                              /dev/nvidia0
1001f000-10020000 rw-s 00000000 00:06 16414                              /dev/nvidia0
10020000-20000000 ---p 00000000 00:00 0 
200000000-200200000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
200200000-200400000 ---p 00000000 00:00 0 
200400000-200404000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
200404000-200600000 ---p 00000000 00:00 0 
200600000-200a00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
200a00000-200e00000 ---p 00000000 00:00 0 
200e00000-200e04000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
200e04000-201000000 ---p 00000000 00:00 0 
201000000-201400000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
201400000-201800000 ---p 00000000 00:00 0 
201800000-201804000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
201804000-201a00000 ---p 00000000 00:00 0 
201a00000-201e00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
201e00000-202200000 ---p 00000000 00:00 0 
202200000-202204000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
202204000-202400000 ---p 00000000 00:00 0 
202400000-202800000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
202800000-202c00000 ---p 00000000 00:00 0 
202c00000-202c04000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
202c04000-202e00000 ---p 00000000 00:00 0 
202e00000-203200000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
203200000-203600000 ---p 00000000 00:00 0 
203600000-203604000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
203604000-203800000 ---p 00000000 00:00 0 
203800000-203c00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
203c00000-204000000 ---p 00000000 00:00 0 
204000000-204004000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
204004000-204200000 ---p 00000000 00:00 0 
204200000-204600000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
204600000-204a00000 ---p 00000000 00:00 0 
204a00000-204a04000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
204a04000-204c00000 ---p 00000000 00:00 0 
204c00000-205000000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
205000000-205400000 ---p 00000000 00:00 0 
205400000-205404000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
205404000-205600000 ---p 00000000 00:00 0 
205600000-205a00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
205a00000-205a04000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
205a04000-205c00000 ---p 00000000 00:00 0 
205c00000-206000000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
206000000-206004000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
206004000-206200000 ---p 00000000 00:00 0 
206200000-206600000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
206600000-206604000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
206604000-206800000 ---p 00000000 00:00 0 
206800000-206c00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
206c00000-206c04000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
206c04000-206e00000 ---p 00000000 00:00 0 
206e00000-207200000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
207200000-207204000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
207204000-207400000 ---p 00000000 00:00 0 
207400000-207800000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
207800000-207804000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
207804000-207a00000 ---p 00000000 00:00 0 
207a00000-207e00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
207e00000-207e04000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
207e04000-208000000 ---p 00000000 00:00 0 
208000000-208400000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
208400000-208600000 ---p 00000000 00:00 0 
208600000-208800000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
208800000-208a00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
208a00000-208c00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
208c00000-208e00000 ---p 00000000 00:00 0 
208e00000-208e04000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
208e04000-209000000 ---p 00000000 00:00 0 
209000000-209400000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
209400000-209800000 ---p 00000000 00:00 0 
209800000-209804000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
209804000-209a00000 ---p 00000000 00:00 0 
209a00000-209e00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
209e00000-20a200000 ---p 00000000 00:00 0 
20a200000-20a204000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20a204000-20a400000 ---p 00000000 00:00 0 
20a400000-20a800000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20a800000-20ac00000 ---p 00000000 00:00 0 
20ac00000-20ac04000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20ac04000-20ae00000 ---p 00000000 00:00 0 
20ae00000-20b200000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20b200000-20b600000 ---p 00000000 00:00 0 
20b600000-20b604000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20b604000-20b800000 ---p 00000000 00:00 0 
20b800000-20bc00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20bc00000-20c000000 ---p 00000000 00:00 0 
20c000000-20c004000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20c004000-20c200000 ---p 00000000 00:00 0 
20c200000-20c600000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20c600000-20ca00000 ---p 00000000 00:00 0 
20ca00000-20ca04000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20ca04000-20cc00000 ---p 00000000 00:00 0 
20cc00000-20d000000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20d000000-20d400000 ---p 00000000 00:00 0 
20d400000-20d404000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20d404000-20d600000 ---p 00000000 00:00 0 
20d600000-20da00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20da00000-20de00000 ---p 00000000 00:00 0 
20de00000-20de04000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20de04000-20e000000 ---p 00000000 00:00 0 
20e000000-20e400000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20e400000-20e404000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20e404000-20e600000 ---p 00000000 00:00 0 
20e600000-20ea00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20ea00000-20ea04000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20ea04000-20ec00000 ---p 00000000 00:00 0 
20ec00000-20f000000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20f000000-20f004000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20f004000-20f200000 ---p 00000000 00:00 0 
20f200000-20f600000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20f600000-20f604000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20f604000-20f800000 ---p 00000000 00:00 0 
20f800000-20fc00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20fc00000-20fc04000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
20fc04000-20fe00000 ---p 00000000 00:00 0 
20fe00000-210200000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
210200000-210204000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
210204000-210400000 ---p 00000000 00:00 0 
210400000-210800000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
210800000-210804000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
210804000-210a00000 ---p 00000000 00:00 0 
210a00000-210e00000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
210e00000-211000000 ---p 00000000 00:00 0 
211000000-211200000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
211200000-211400000 rw-s 00000000 00:06 16413                            /dev/nvidiactl
211400000-300200000 ---p 00000000 00:00 0 
10000000000-10204400000 ---p 00000000 00:00 0 
10204400000-10204600000 rw-s 00000000 00:05 42643                        /dev/zero (deleted)
10204600000-10204800000 rw-s 00000000 00:06 16413                        /dev/nvidiactl
10204800000-10204a00000 rw-s 00000000 00:05 42644                        /dev/zero (deleted)
10204a00000-10204c00000 rw-s 00000000 00:06 16413                        /dev/nvidiactl
10204c00000-10204ed6000 rw-s 00000000 00:06 16413                        /dev/nvidiactl
10204ed6000-1030ae00000 ---p 00000000 00:00 0 
1030ae00000-1030b000000 rw-s 00000000 00:05 42648                        /dev/zero (deleted)
1030b000000-1030b200000 rw-s 00000000 00:06 16413                        /dev/nvidiactl
1030b200000-1030b400000 rw-s 00000000 00:05 42649                        /dev/zero (deleted)
1030b400000-1030b600000 rw-s 00000000 00:06 16413                        /dev/nvidiactl
1030b600000-1030b8d6000 rw-s 00000000 00:06 16413                        /dev/nvidiactl
1030b8d6000-1038ba00000 ---p 00000000 00:00 0 Aborted (core dumped)

Hi,

Do you also meet this error on our default sample?
Or it only comes with a customized model?

We want to check if this is a environment setting issue or is from TensorRT libraries.
Thanks.

I also meet this error on NVIDIA’s default sample.
if you already have solved this problem, would let me know how to solve??

Hi,

We didn’t meet this issue in our environment before.
Could you share more information about your desktop setting?
EX. GPU name, OS version, CUDA version, cuDNN version.

Thanks.