Please provide complete information as applicable to your setup.
• Hardware Platform (Jetson / GPU): Quadro RTX 4000
• DeepStream Version: 6.1.1
• TensorRT Version: 8.5.3.1
• NVIDIA GPU Driver Version (valid for GPU only): 510.47.03
I experimented on the deepstream_image_meta_test sample. I used CMakeLists to compile the project with -fsanitize option to to enable LeakSanitizer profiler.
cmake_minimum_required(VERSION 3.15 FATAL_ERROR)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED True)
set(CMAKE_CUDA_COMPILER /usr/local/cuda-11.5/bin/nvcc)
project(checkLeakMem LANGUAGES CXX CUDA)
# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -Wall -pthread")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -fno-omit-frame-pointer -fsanitize=leak -g -pthread")
set(CMAKE_CXX_STANDARD 17)
find_package(PkgConfig REQUIRED)
add_executable(${PROJECT_NAME} main.cpp)
pkg_check_modules(Uuid REQUIRED IMPORTED_TARGET uuid)
include_directories(${Uuid_INCLUDE_DIRS})
target_link_libraries(${PROJECT_NAME} ${Uuid_LIBRARIES})
# cuda
find_package(CUDA REQUIRED)
set(CMAKE_CUDA_ARCHITECTURES 52 60 61 75 CACHE STRING "CUDA architectures")
set_target_properties(${PROJECT_NAME} PROPERTIES CUDA_STANDARD 17)
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -O3 --compiler-options -fno-strict-aliasing --ptxas-options=-v --generate-line-info")
include_directories(${CUDA_INCLUDE_DIRS})
target_link_libraries(${PROJECT_NAME} ${CUDA_LIBRARIES})
# gstreamer
find_package(PkgConfig REQUIRED)
pkg_check_modules(GST_PKG REQUIRED IMPORTED_TARGET gstreamer-app-1.0)
target_include_directories(${PROJECT_NAME} PUBLIC ${GST_PKG_INCLUDE_DIRS})
target_link_libraries(${PROJECT_NAME} PkgConfig::GST_PKG)
target_compile_options(${PROJECT_NAME} PUBLIC ${GST_PKG_CFLAGS_OTHER})
# deepstream
target_include_directories(${PROJECT_NAME} PUBLIC /opt/nvidia/deepstream/deepstream/sources/includes)
target_link_directories(${PROJECT_NAME} PUBLIC /opt/nvidia/deepstream/deepstream/lib)
target_link_libraries(${PROJECT_NAME} -lnvbufsurface -lnvbufsurftransform -lnvdsgst_helper -lnvds_batch_jpegenc
-lnvdsgst_meta -lnvds_meta -lnvinfer -lnvinfer_plugin -lnvds_inferutils
-lnvds_utils -lm -lnvds_infercustomparser)
# OpenCV
find_package(OpenCV REQUIRED)
include_directories(${OpenCV_INCLUDE_DIRS})
target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS})
target_link_libraries(${PROJECT_NAME} -lstdc++fs -lcurl)
# Json-glib
pkg_check_modules(JSON-GLIB REQUIRED json-glib-1.0)
include_directories(${JSON-GLIB_INCLUDE_DIRS})
target_link_libraries(${PROJECT_NAME} ${JSON-GLIB_LIBRARIES})
The result is shown below indicating that there are a multiple direct leaks
==18434==ERROR: LeakSanitizer: detected memory leaks
Direct leak of 6656 byte(s) in 26 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc456fcb8ed (/usr/lib/x86_64-linux-gnu/libfontconfig.so.1+0x1d8ed)
Direct leak of 520 byte(s) in 13 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc3a926afe1 (<unknown module>)
Direct leak of 216 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc4532672a7 (/usr/lib/x86_64-linux-gnu/libnvparsers.so.8.4.1+0x32a7)
#2 0x31d6ffffffff (<unknown module>)
Direct leak of 192 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc4708bda85 in allocate_buffers (/opt/nvidia/deepstream/deepstream/lib/libnvds_batch_jpegenc.so+0x1a85)
#2 0x7fc4708be2fd in nvds_obj_enc_process (/opt/nvidia/deepstream/deepstream/lib/libnvds_batch_jpegenc.so+0x22fd)
#3 0x563b3141d888 in pgie_src_pad_buffer_probe /home/martin/minhbq6/checkLeakMem/main.cpp:241
#4 0x7fc46fc7b287 (/usr/lib/x86_64-linux-gnu/libgstreamer-1.0.so.0+0x74287)
Direct leak of 184 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9251b54 (<unknown module>)
Direct leak of 112 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9261b3b (<unknown module>)
Direct leak of 100 byte(s) in 3 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc3a9262b2e (<unknown module>)
Direct leak of 40 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9264a33 (<unknown module>)
Direct leak of 24 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc3a9264931 (<unknown module>)
Direct leak of 8 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e2c73 in realloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xec73)
#1 0x7fc3a9261811 (<unknown module>)
Direct leak of 8 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc4517b526c (<unknown module>)
Indirect leak of 7135 byte(s) in 13 object(s) allocated from:
#0 0x7fc4710e2c73 in realloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xec73)
#1 0x7fc3a9264d7c (<unknown module>)
Indirect leak of 1632 byte(s) in 51 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc456fcbfd8 (/usr/lib/x86_64-linux-gnu/libfontconfig.so.1+0x1dfd8)
Indirect leak of 808 byte(s) in 2 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9264b4d (<unknown module>)
Indirect leak of 736 byte(s) in 4 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9251b54 (<unknown module>)
Indirect leak of 519 byte(s) in 50 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc46f2f6a29 in __strdup (/lib/x86_64-linux-gnu/libc.so.6+0x9da29)
Indirect leak of 264 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a924667d (<unknown module>)
Indirect leak of 240 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9243a57 (<unknown module>)
#2 0x7ffffffff (<unknown module>)
Indirect leak of 232 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9260bdf (<unknown module>)
Indirect leak of 224 byte(s) in 4 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc3a924b35d (<unknown module>)
Indirect leak of 176 byte(s) in 2 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9259b1a (<unknown module>)
Indirect leak of 160 byte(s) in 4 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc3a926afe1 (<unknown module>)
Indirect leak of 128 byte(s) in 2 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a92610b8 (<unknown module>)
Indirect leak of 112 byte(s) in 6 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc3a9264fc3 (<unknown module>)
Indirect leak of 112 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9261b3b (<unknown module>)
Indirect leak of 96 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a92566ff (<unknown module>)
Indirect leak of 96 byte(s) in 2 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc3a926b101 (<unknown module>)
Indirect leak of 80 byte(s) in 2 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9264a33 (<unknown module>)
Indirect leak of 80 byte(s) in 2 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc3a926af8a (<unknown module>)
Indirect leak of 64 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e2c73 in realloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xec73)
#1 0x7fc3a92569dd (<unknown module>)
#2 0xeb5c5c5f623a02ff (<unknown module>)
Indirect leak of 64 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a92584d7 (<unknown module>)
Indirect leak of 48 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9246637 (<unknown module>)
Indirect leak of 48 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9246648 (<unknown module>)
Indirect leak of 48 byte(s) in 2 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc3a9264931 (<unknown module>)
Indirect leak of 42 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc3a9265289 (<unknown module>)
#2 0xeb5c5c5f623a02ff (<unknown module>)
Indirect leak of 24 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a92588b2 (<unknown module>)
Indirect leak of 24 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc3a9264995 (<unknown module>)
Indirect leak of 24 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e37e3 (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xf7e3)
#1 0x7fc3a9244601 (<unknown module>)
Indirect leak of 16 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e2c73 in realloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xec73)
#1 0x7fc3a9261981 (<unknown module>)
Indirect leak of 7 byte(s) in 1 object(s) allocated from:
#0 0x7fc4710e2acb in __interceptor_malloc (/usr/lib/x86_64-linux-gnu/liblsan.so.0+0xeacb)
#1 0x7fc3a9262b2e (<unknown module>)
SUMMARY: LeakSanitizer: 21299 byte(s) leaked in 211 allocation(s).
There are many 1 object leaks that contribute constant and insignificant leak. However, the most amount of memory leak is found in liblsan.so with the number of object being leaked increase over time. When testing on rtspsrc, after few hours of operating, the memory leaks can accumulate up to 60GB. Can anyone suggest how to fix this problem?