Fatal and panic error using nvurisrcbin

Please provide complete information as applicable to your setup.

• Hardware Platform (Jetson / GPU) : RTX 5060
• DeepStream Version : 8.0
• TensorRT Version : 10.15.1
• NVIDIA GPU Driver Version : 590.48.01
• Issue Type : Bug
• How to reproduce the issue ?

I try to start a simple Pipeline :

nvurisrcbin → fakesink

My source :

  g_print("Create pipeline\n");
  GstElement *pipeline = gst_pipeline_new("face-pipeline");
  /* Create gstreamer elements */
  g_print("Create element\n");
  GstElement *source, *fakesink;
  source = gst_element_factory_make("nvurisrcbin", "multi-urisrcbin");
  g_object_set(
      G_OBJECT(source), "uri",
      "rtsp://192.168.1.113:8554/live/3687476a-a0fb-40b6-8662-74329519cd1f",
      // "file:///opt/nvidia/deepstream/deepstream-8.0/samples/streams/"
      NULL);

  fakesink = gst_element_factory_make("fakesink", "sink");

  g_print("add bus...\n");
  /* we add a message handler */
  bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline));
  bus_watch_id = gst_bus_add_watch(bus, bus_call, loop);
  gst_object_unref(bus);

  g_print("Bind element\n");
  gst_bin_add_many(GST_BIN(pipeline), source, fakesink, NULL);
  gst_element_link_many(source, fakesink, NULL);

  /* Set the pipeline to "playing" state */
  g_print("Using file: %s\n", argv[1]);
  gst_element_set_state(pipeline, GST_STATE_PLAYING);

  std::signal(SIGINT, on_SIGINT);

  /* Iterate */
  g_print("Running...\n");
  g_main_loop_run(loop);

When I start a pipeline with an nvurisrcbin element using rtsp uri, deepstream crash with this log.

Protocol set: 0x7
Running…
FATAL 56: uncaught error
PANIC 56: uncaught error (calling abort)
Abandon (core dumped)

The error occure when using nvurisrcbin and nvmultiurisrcbin with RTSP, file source work.

The most strange thinks is the error occure only when I launch a pipeline in an CPP project using CMake. I can start a working pipeline in the cli with this command :

gst-launch-1.0 nvmultiurisrcbin width=640 height=480 live-source=1 batched-push-timeout=33333 max-batch-size=8 port=9999 uri-list=rtsp://192.168.1.113:8554/live/3687476a-a0fb-40b6-8662-74329519cd1f ! fakesink

Also, work fine when I change the test1 to recreate this pipeline :

  pipeline = gst_pipeline_new("dstest1-pipeline");

  source = gst_element_factory_make("nvmultiurisrcbin", "file-source");
  g_object_set(
      G_OBJECT(source), "uri-list",
      "rtsp://192.168.1.113:8554/live/3687476a-a0fb-40b6-8662-74329519cd1f",
      "ip-address", "0.0.0.0", "port", "9999", "live-source", 1,
      "batched-push-timeout", 33333, "max-batch-size", 8, "width", 640,
      "height", 480, NULL);
  sink = gst_element_factory_make("fakesink", "nv3d-sink");

  bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline));
  bus_watch_id = gst_bus_add_watch(bus, bus_call, loop);
  gst_object_unref(bus);

  gst_bin_add_many(GST_BIN(pipeline), source, sink, NULL);
  g_print("Added elements to bin\n");

  if (!gst_element_link_many(source, sink, NULL)) {
    g_printerr("Elements could not be linked: 2. Exiting.\n");
    return -1;
  }

  g_print("Using file: %s\n", argv[1]);
  gst_element_set_state(pipeline, GST_STATE_PLAYING);

  g_print("Running...\n");
  g_main_loop_run(loop);

For info, this is my CMakeLists.txt, it work fine with other ds gst element and plugin.

cmake_minimum_required(VERSION 3.25)
project(deepstream-test2 LANGUAGES C CXX) #project(deepstream-test5 LANGUAGES CUDA C CXX)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED on)
set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_BUILD_TYPE Debug)
find_package(PkgConfig REQUIRED)
pkg_check_modules(PKGS REQUIRED uuid x11 gstreamer-video-1.0 json-glib-1.0 gstreamer-1.0 gstreamer-rtsp-server-1.0 yaml-cpp)
find_package(CUDAToolkit)
message("CUDA_VER is ${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR}")

if(NOT DEFINED CMAKE_CUDA_STANDARD)
    set(CMAKE_CUDA_STANDARD 11)
    set(CMAKE_CUDA_STANDARD_REQUIRED True)
endif()

SET_SOURCE_FILES_PROPERTIES(
  ${OBJS}
  PROPERTIES
  EXTERNAL_OBJECT true
  GENERATED true
)

if ( "${CMAKE_BUILD_TYPE}" STREQUAL "")
    set(CMAKE_BUILD_TYPE "Debug")
endif ()

include_guard(GLOBAL)

include(FetchContent)

FetchContent_Declare(
    milvus-sdk
    GIT_REPOSITORY    https://github.com/milvus-io/milvus-sdk-cpp.git
    GIT_TAG           2.6
    GIT_SHALLOW       TRUE
    GIT_PROGRESS      TRUE
    UPDATE_DISCONNECTED TRUE
)

FetchContent_MakeAvailable(milvus-sdk)

message(STATUS "  milvus-sdk_SOURCE_DIR: ${milvus-sdk_SOURCE_DIR}")
message(STATUS "  milvus-sdk_BINARY_DIR: ${milvus-sdk_BINARY_DIR}")

include_directories(${milvus-sdk_SOURCE_DIR}/src/include)
link_directories(${milvus-sdk_BINARY_DIR}/src)

add_executable(${PROJECT_NAME}
    ${OBJS}
    deepstream_test2_app.cpp
    face_registry.cpp
    gst_setup.cpp
    MilvusContext.cpp
)

target_include_directories(${PROJECT_NAME} PRIVATE
    ${PKGS_INCLUDE_DIRS}
    ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
    /opt/nvidia/deepstream/deepstream-8.0/sources/apps/apps-common/includes
    /opt/nvidia/deepstream/deepstream-8.0/sources/includes
    /usr/include/jsoncpp
    /usr/local/include/milvus
    ${CMAKE_CURRENT_SOURCE_DIR}/cparse
    ${CMAKE_CURRENT_SOURCE_DIR}
)

target_link_libraries(${PROJECT_NAME} PRIVATE
    ${PKGS_LIBRARIES}
    CUDA::cuda_driver
    nvds_utils nvdsgst_helper m nvdsgst_meta nvds_meta stdc++
    nvbufsurface nvbufsurftransform nvds_batch_jpegenc nvds_rest_server uuid jpeg png nvds_yml_parser
    nvdsgst_smartrecord nvds_msgbroker nvdsgst_customhelper
    pthread jsoncpp gstrtspserver-1.0
    milvus_sdk
    PRIVATE CUDA::cudart
)

target_link_directories(${PROJECT_NAME} PRIVATE
    /opt/nvidia/deepstream/deepstream-8.0/lib/
)

target_link_options(${PROJECT_NAME} PRIVATE -Wl,-rpath,/opt/nvidia/deepstream/deepstream-8.0/lib/)

Thank for your help.

I did a more in-depth analysis, I found the error is throw by gstrtspsrc, when I replace the nvurisrcbin by rtspsrc element, the same error occured.

Again, by comparing the debug log, I can say the error come when rtspsrc “retreive_sdp”.

The crash occur before this log :

0:00:00.234106928 284221 0x76141c000fb0 DEBUG                rtspsrc gstrtspsrc.c:8316:gst_rtspsrc_retrieve_sdp:<src> create options... (async)
0:00:00.234121564 284221 0x76141c000fb0 DEBUG                rtspsrc gstrtspsrc.c:8325:gst_rtspsrc_retrieve_sdp:<src> send options...

Here, it’s the last 47 lines of debug logs.

0:00:00.224873398 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x577056e75ef0, state-changed from fakesink0, type mask is 4294967295
0:00:00.224879578 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x577056e75ef0, time 99:99:99.999999999, seq-num 4, element 'fakesink0', GstMessageStateChanged, old-state=(GstState)null, new-state=(GstState)ready, pending-state=(GstState)void-pending;
0:00:00.224884032 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.224888500 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x577057014ab0, element from dsnvmultiuribin0_creator, type mask is 4294967295
0:00:00.224893977 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with element message: 0x577057014ab0, time 99:99:99.999999999, seq-num 8, element 'dsnvmultiuribin0_creator', stream-add, source-id=(uint)0, sensor-id=(string)NULL, sensor-name=(string)NULL, uri=(string)rtsp://192.168.1.166:8554/mystream;
0:00:00.224896341 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.224902895 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x761410012370, state-changed from src_bin_muxer, type mask is 4294967295
0:00:00.224907046 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x761410012370, time 99:99:99.999999999, seq-num 12, element 'src_bin_muxer', GstMessageStateChanged, old-state=(GstState)null, new-state=(GstState)ready, pending-state=(GstState)void-pending;
0:00:00.224909139 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.224912823 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x76141932a7a0, state-changed from src, type mask is 4294967295
0:00:00.224916080 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x76141932a7a0, time 99:99:99.999999999, seq-num 13, element 'src', GstMessageStateChanged, old-state=(GstState)null, new-state=(GstState)ready, pending-state=(GstState)void-pending;
0:00:00.224917958 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.224920897 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x76141932b620, state-changed from dsnvurisrcbin0, type mask is 4294967295
0:00:00.224924328 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x76141932b620, time 99:99:99.999999999, seq-num 14, element 'dsnvurisrcbin0', GstMessageStateChanged, old-state=(GstState)null, new-state=(GstState)ready, pending-state=(GstState)void-pending;
0:00:00.224926364 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.224929388 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x761410013140, state-changed from dsnvmultiuribin0_creator, type mask is 4294967295
0:00:00.224932816 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x761410013140, time 99:99:99.999999999, seq-num 15, element 'dsnvmultiuribin0_creator', GstMessageStateChanged, old-state=(GstState)null, new-state=(GstState)ready, pending-state=(GstState)void-pending;
0:00:00.224935828 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.224938790 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x57705704abd0, state-changed from dsnvmultiuribin0, type mask is 4294967295
0:00:00.224942890 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x57705704abd0, time 99:99:99.999999999, seq-num 16, element 'dsnvmultiuribin0', GstMessageStateChanged, old-state=(GstState)null, new-state=(GstState)ready, pending-state=(GstState)void-pending;
0:00:00.224945043 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.224948255 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x57705704ad90, state-changed from pipeline0, type mask is 4294967295
0:00:00.224951366 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x57705704ad90, time 99:99:99.999999999, seq-num 17, element 'pipeline0', GstMessageStateChanged, old-state=(GstState)null, new-state=(GstState)ready, pending-state=(GstState)paused;
0:00:00.224953755 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.224956696 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x57705701dc80, state-changed from src_bin_muxer, type mask is 4294967295
0:00:00.224961237 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x57705701dc80, time 99:99:99.999999999, seq-num 19, element 'src_bin_muxer', GstMessageStateChanged, old-state=(GstState)ready, new-state=(GstState)paused, pending-state=(GstState)void-pending;
0:00:00.224965702 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.224969826 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x761410014050, progress from src, type mask is 4294967295
0:00:00.224974671 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with progress message: 0x761410014050, time 99:99:99.999999999, seq-num 21, element 'src', GstMessageProgress, type=(GstProgressType)start, code=(string)open, text=(string)"Opening\ Stream", percent=(int)0, timeout=(int)-1;
0:00:00.224982762 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.224987678 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x761410014210, state-changed from src, type mask is 4294967295
0:00:00.224991110 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x761410014210, time 99:99:99.999999999, seq-num 22, element 'src', GstMessageStateChanged, old-state=(GstState)ready, new-state=(GstState)paused, pending-state=(GstState)void-pending;
0:00:00.224994623 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.224998698 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x761410013e50, state-changed from dsnvurisrcbin0, type mask is 4294967295
0:00:00.225003700 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x761410013e50, time 99:99:99.999999999, seq-num 23, element 'dsnvurisrcbin0', GstMessageStateChanged, old-state=(GstState)ready, new-state=(GstState)paused, pending-state=(GstState)void-pending;
0:00:00.225006323 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.225011219 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x761124002c20, progress from src, type mask is 4294967295
0:00:00.225017699 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with progress message: 0x761124002c20, time 99:99:99.999999999, seq-num 24, element 'src', GstMessageProgress, type=(GstProgressType)continue, code=(string)connect, text=(string)"Connecting\ to\ rtsp://192.168.1.166:8554/mystream", percent=(int)0, timeout=(int)-1;
0:00:00.225025042 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.225031096 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x577057049c60, state-changed from dsnvmultiuribin0_creator, type mask is 4294967295
0:00:00.225035223 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x577057049c60, time 99:99:99.999999999, seq-num 25, element 'dsnvmultiuribin0_creator', GstMessageStateChanged, old-state=(GstState)ready, new-state=(GstState)paused, pending-state=(GstState)void-pending;
0:00:00.225037402 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.225040509 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x57705704b7c0, state-changed from dsnvmultiuribin0, type mask is 4294967295
0:00:00.225045606 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x57705704b7c0, time 99:99:99.999999999, seq-num 26, element 'dsnvmultiuribin0', GstMessageStateChanged, old-state=(GstState)ready, new-state=(GstState)paused, pending-state=(GstState)void-pending;
0:00:00.225048614 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
0:00:00.225058056 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:573:gst_bus_timed_pop_filtered:<bus2> got message 0x57705701f840, state-changed from pipeline0, type mask is 4294967295
0:00:00.225063182 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:831:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 calling dispatch with state-changed message: 0x57705701f840, time 99:99:99.999999999, seq-num 27, element 'pipeline0', GstMessageStateChanged, old-state=(GstState)ready, new-state=(GstState)paused, pending-state=(GstState)void-pending;
0:00:00.225072481 284221 0x577056b94ac0 DEBUG                GST_BUS gstbus.c:837:gst_bus_source_dispatch:<bus2> source 0x577056e9dde0 handler returns 1
FATAL 56: uncaught error
PANIC 56: uncaught error (calling abort)
Abandon (core dumped)

Hello, again I have more information.

I have rework the CMakeLists to remove some libraries I didn’t use and to target if the error occured from a library and I found somethink.

Here it’s my new CMakeLists.txt

cmake_minimum_required(VERSION 3.25)
project(deepstream-test2 LANGUAGES C CXX)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED on)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_BUILD_TYPE Debug)
find_package(PkgConfig REQUIRED)
pkg_check_modules(PKGS REQUIRED gstreamer-video-1.0 json-glib-1.0 gstreamer-1.0 gstreamer-rtsp-server-1.0 yaml-cpp)
find_package(CUDAToolkit)
message("CUDA_VER is ${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR}")

if(NOT DEFINED CMAKE_CUDA_STANDARD)
    set(CMAKE_CUDA_STANDARD 11)
    set(CMAKE_CUDA_STANDARD_REQUIRED True)
endif()

SET_SOURCE_FILES_PROPERTIES(
  ${OBJS}
  PROPERTIES
  EXTERNAL_OBJECT true
  GENERATED true
)

if ( "${CMAKE_BUILD_TYPE}" STREQUAL "")
    set(CMAKE_BUILD_TYPE "Debug")
endif ()

include_guard(GLOBAL)

include(FetchContent)

FetchContent_Declare(
    milvus-sdk
    GIT_REPOSITORY    https://github.com/milvus-io/milvus-sdk-cpp.git
    GIT_TAG           2.6
    GIT_SHALLOW       TRUE
    GIT_PROGRESS      TRUE
    UPDATE_DISCONNECTED TRUE
)

FetchContent_MakeAvailable(milvus-sdk)

message(STATUS "  milvus-sdk_SOURCE_DIR: ${milvus-sdk_SOURCE_DIR}")
message(STATUS "  milvus-sdk_BINARY_DIR: ${milvus-sdk_BINARY_DIR}")

include_directories(${milvus-sdk_SOURCE_DIR}/src/include)
link_directories(${milvus-sdk_BINARY_DIR}/src)

add_executable(${PROJECT_NAME}
    ${OBJS}
    deepstream_test2_app.cpp
    face_registry.cpp
    gst_setup.cpp
    MilvusContext.cpp
    rest_server_callbacks.cpp
)

target_include_directories(${PROJECT_NAME} PRIVATE
    ${PKGS_INCLUDE_DIRS}
    ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
    /opt/nvidia/deepstream/deepstream-8.0/sources/apps/apps-common/includes
    /opt/nvidia/deepstream/deepstream-8.0/sources/includes
    /usr/include/jsoncpp
    /usr/local/include/milvus
    ${CMAKE_CURRENT_SOURCE_DIR}
)

target_link_libraries(${PROJECT_NAME} PRIVATE
    ${PKGS_LIBRARIES}
    CUDA::cuda_driver
    m nvdsgst_meta nvds_meta stdc++
    nvds_yml_parser
    nvdsgst_customhelper
    pthread jsoncpp gstrtspserver-1.0
    milvus_sdk
    PRIVATE CUDA::cudart
    nvds_rest_server
)

target_link_directories(${PROJECT_NAME} PRIVATE
    /opt/nvidia/deepstream/deepstream-8.0/lib/
)

target_link_options(${PROJECT_NAME} PRIVATE 
    -Wl,-rpath,/opt/nvidia/deepstream/deepstream-8.0/lib/
)

I have identify if I remove the nvds_rest_server from target_link_libraries, the rtsp work fine.

So, I can run the app with RTSP input but without REST server, and I need an REST server on my app. So to “merge” the multiurisrcbin with the REST server, I have follow the sample_app “deepstream_server”, but the error don’t disappear.

To summarize,

the error occured when I use both nvds_rest_server and a rtsp source. Regardless I use a nvmultiurisrcbin, nvurisrcbin or rtspsrc source element.

Based on your description, I built a program but am unable to reproduce the error. you can refer to this sample code.

CMakeLists.txt (3.5 KB)
main.cpp (14.8 KB)

Without REST server linked

cmake -DENABLE_REST_SERVER=OFF .. && make

Hello,

Thank you for the response and the documentation of the source.

Unfortunately, the error persists on my computer. When I run an application with an REST server and a RTSP source, the error appeared.

To be sure, I have do an update “apt update && apt upgrade”, also uninstall and reinstall deepstream.

Based on your source, I have push the analyse and found an interesting fact.

If I build the app with rest_server but don’t start it, the error is throw.

The exemple here :

#include "nvds_rest_server.h"
#include <gst/gst.h>

struct RestServerCtx {
  NvDsRestServer *server;
  NvDsServerConfig conf;
};

static RestServerCtx *start_rest_server(const std::string &ip,
                                        const std::string &port) {
  auto *ctx = new RestServerCtx();
  ctx->conf.ip = ip;
  ctx->conf.port = port;

  NvDsServerCallbacks cbs = {};

  /* Minimal stream callback – just logs the request */
  cbs.stream_cb = [](NvDsServerStreamInfo *info, void *) {
    g_print("[REPRO][REST] stream_cb: camera_url=%s change=%s\n",
            info->value_camera_url.c_str(), info->value_change.c_str());
    info->status = STREAM_ADD_SUCCESS;
    info->err_info.code = StatusOk;
  };

  g_print("[REPRO] Starting REST server on %s:%s ...\n", ip.c_str(),
          port.c_str());

  ctx->server = nvds_rest_server_start(&ctx->conf, &cbs);
  if (!ctx->server) {
    g_printerr("[REPRO] Failed to start REST server!\n");
    delete ctx;
    return nullptr;
  }

  g_print("[REPRO] REST server started successfully.\n");
  return ctx;
}

static void stop_rest_server(RestServerCtx *ctx) {
  if (ctx && ctx->server) {
    g_print("[REPRO] Stopping REST server...\n");
    nvds_rest_server_stop(ctx->server);
    delete ctx;
  }
}

int main(int argc, char *argv[]) {
  gst_init(&argc, &argv);
  GMainLoop *loop = g_main_loop_new(NULL, FALSE);
  GstElement *pipeline = gst_pipeline_new("pipeline");
  GstElement *source = gst_element_factory_make("rtspsrc", "dsnvmultiuribin0");
  g_object_set(G_OBJECT(source), "location",
               "rtsp://192.168.1.166:8554/mystream", NULL);
  GstElement *fakesink = gst_element_factory_make("fakesink", "fakesink0");
  gst_bin_add_many(GST_BIN(pipeline), source, fakesink, NULL);
  gst_element_link_many(source, fakesink, NULL);

  //----------Don't start the server--------//
  // RestServerCtx *ctx = start_rest_server("localhost", "8080");
  gst_element_set_state(pipeline, GST_STATE_PLAYING);
  g_main_loop_run(loop);

  //----------Don't start the server--------//
  // stop_rest_server(ctx);
  gst_element_set_state(pipeline, GST_STATE_NULL);
  gst_object_unref(GST_OBJECT(pipeline));
  g_main_loop_unref(loop);
  return 0;
}
(base) user@gpuserver00:~/source/test/ds_crash/build$ make
[ 50%] Building CXX object CMakeFiles/deepstream-crash.dir/main.cpp.o
[100%] Linking CXX executable deepstream-crash
[100%] Built target deepstream-crash
(base) user@gpuserver00:~/source/test/ds_crash/build$ ./deepstream-crash
FATAL 56: uncaught error
PANIC 56: uncaught error (calling abort)
Abandon (core dumped)

I think the error is from my computer but I don’t know what is supposed to do this and how to investigate.

If you have some suggestion, I’m open of all solution.

Another discovery, I start the same sample in a deepstream docker container and I got the same error.

This is very strange. Did you install apt install libcivetweb-dev separately? Since it only loads the .so file and doesn’t call it, I suspect the error might be caused by the following code snippet.

// in /opt/nvidia/deepstream/deepstream/sources/libs/nvds_rest_server/nvds_rest_server.cpp
void __attribute__((constructor)) nvds_rest_server_init (void)
{
  mg_init_library (0);
}

What is the output from the following command line?

find / -name "libcivetweb.so*" 2>/dev/null
  1. Try to recompile the nvds_rest_server library
  2. Or try before executing your program:
export LD_LIBRARY_PATH=/opt/nvidia/deepstream/deepstream/lib:$LD_LIBRARY_PATH

Try using gdb to get the stack trace

$ gdb --args your_app xxx
$ bt

I haven’t install other civetweb

(base) user@gpuserver00:~/source/test/ds_crash/build$ find / -name "libcivetweb.so*" 2>/dev/null
/opt/nvidia/deepstream/deepstream-8.0/lib/libcivetweb.so.1.16.0
/opt/nvidia/deepstream/deepstream-8.0/lib/libcivetweb.so
/opt/nvidia/deepstream/deepstream-8.0/lib/libcivetweb.so.1
(base) user@gpuserver00:~/source/test/ds_crash/build$

I have recompile nvds_rest_server and add lib in LD_LIBRARY_PATH but same error.

The output of gdb say No stack. I don’t know how to use it, possiblely I have do a mistake :

(base) user@gpuserver00:~/source/test/DeepVision/deepstream-test2/build$ echo $LD_LIBRARY_PATH
/opt/nvidia/deepstream/deepstream-8.0/lib:
(base) user@gpuserver00:~/source/test/DeepVision/deepstream-test2/build$ gdb --args ./deepstream-rtsp-rest-crash-repro --uri rtsp://192.168.1.166:8554/mystream --source-type rtspsrc
GNU gdb (Ubuntu 15.0.50.20240403-0ubuntu1) 15.0.50.20240403-git
Copyright (C) 2024 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<https://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
    <http://www.gnu.org/software/gdb/documentation/>.

For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from ./deepstream-rtsp-rest-crash-repro...
(gdb) bt
No stack.
(gdb)

For information, I don’t know if it’s normal but the directory /opt/nvidia/deepstream/deepstream/lib/ contain only kafka library, I have update cmake file to point /opt/nvidia/deepstream/deepstream-8.0/lib/ thas contain all deepstream lib.

Run the application first, wait the core dump.

$ r
xxxx
$ bt

How did you start the container? Was it with parameters like the one below?

docker run --gpus all -it --rm --runtime=nvidia --network=host --privileged -v
/tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=$DISPLAY -w
/opt/nvidia/deepstream/deepstream-8.0 nvcr.io/nvidia/
deepstream:8.0-triton-multiarch

If you run this sample within a container without making any modifications, will it crash?
/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-server/

#0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>) at ./nptl/pthread_kill.c:44
#1  __pthread_kill_internal (signo=6, threadid=<optimized out>) at ./nptl/pthread_kill.c:78
#2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6) at ./nptl/pthread_kill.c:89
#3  0x00007ffff704527e in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#4  0x00007ffff70288ff in __GI_abort () at ./stdlib/abort.c:79
#5  0x00007ffff6f6db4c in duk_default_panic_handler () at /opt/nvidia/deepstream/deepstream-8.0/lib/libcivetweb.so.1
#6  0x00007ffff6f6db9f in duk_default_fatal_handler () at /opt/nvidia/deepstream/deepstream-8.0/lib/libcivetweb.so.1
#7  0x00007ffff6f70261 in duk_fatal () at /opt/nvidia/deepstream/deepstream-8.0/lib/libcivetweb.so.1
#8  0x00007ffff6f702b1 in duk_err_longjmp () at /opt/nvidia/deepstream/deepstream-8.0/lib/libcivetweb.so.1
#9  0x00007ffff6f7837d in duk_err_create_and_throw () at /opt/nvidia/deepstream/deepstream-8.0/lib/libcivetweb.so.1
#10 0x00007ffff6f78d54 in duk_err_handle_error () at /opt/nvidia/deepstream/deepstream-8.0/lib/libcivetweb.so.1
#11 0x00007ffff6f93ef5 in duk.do_compile () at /opt/nvidia/deepstream/deepstream-8.0/lib/libcivetweb.so.1
#12 0x00007ffff6f9419e in duk_compile_raw () at /opt/nvidia/deepstream/deepstream-8.0/lib/libcivetweb.so.1
#13 0x00007ffff6f94ceb in duk_eval_raw () at /opt/nvidia/deepstream/deepstream-8.0/lib/libcivetweb.so.1
#14 0x00007ffff47bf89d in ??? () at /usr/lib/x86_64-linux-gnu/libproxy/libpxbackend-1.0.so
#15 0x00007ffff7e22ff9 in g_type_create_instance () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#16 0x00007ffff7e08a64 in ??? () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#17 0x00007ffff7e0a016 in g_object_new_with_properties () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#18 0x00007ffff7e0af71 in g_object_new () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#19 0x00007ffff47c20f8 in ??? () at /usr/lib/x86_64-linux-gnu/libproxy/libpxbackend-1.0.so
#20 0x00007ffff7e08bba in ??? () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#21 0x00007ffff7e0ae7d in g_object_new_valist () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#22 0x00007ffff47bfcf3 in px_manager_new_with_options () at /usr/lib/x86_64-linux-gnu/libproxy/libpxbackend-1.0.so
#23 0x00007ffff67b72cf in px_proxy_factory_new () at /lib/x86_64-linux-gnu/libproxy.so.1
#24 0x00007ffff67bd7bd in g_libproxy_resolver_init (resolver=0x7fff200408f0) at ../proxy/libproxy/glibproxyresolver.c:86
#25 g_libproxy_resolver_init (resolver=0x7fff200408f0) at ../proxy/libproxy/glibproxyresolver.c:83
#26 0x00007ffff7e22ff9 in g_type_create_instance () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#27 0x00007ffff7e08a64 in ??? () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#28 0x00007ffff7e0a016 in g_object_new_with_properties () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#29 0x00007ffff7e0af71 in g_object_new () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#30 0x00007ffff661eadc in ??? () at /lib/x86_64-linux-gnu/libgio-2.0.so.0
#31 0x00007ffff6624596 in ??? () at /lib/x86_64-linux-gnu/libgio-2.0.so.0
--Type <RET> for more, q to quit, c to continue without paging--
#32 0x00007ffff6632f6a in g_proxy_resolver_get_default () at /lib/x86_64-linux-gnu/libgio-2.0.so.0
#33 0x00007ffff66330d5 in ??? () at /lib/x86_64-linux-gnu/libgio-2.0.so.0
#34 0x00007ffff7e08360 in ??? () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#35 0x00007ffff7e08b98 in ??? () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#36 0x00007ffff7e0abc3 in g_object_new_valist () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#37 0x00007ffff7e0af4f in g_object_new () at /lib/x86_64-linux-gnu/libgobject-2.0.so.0
#38 0x00007ffff662b1a9 in ??? () at /lib/x86_64-linux-gnu/libgio-2.0.so.0
#39 0x00007ffff664bee5 in g_socket_client_connect () at /lib/x86_64-linux-gnu/libgio-2.0.so.0
#40 0x00007ffff664c0bf in g_socket_client_connect_to_uri () at /lib/x86_64-linux-gnu/libgio-2.0.so.0
#41 0x00007ffff67a3638 in gst_rtsp_connection_connect_with_response_usec () at /lib/x86_64-linux-gnu/libgstrtsp-1.0.so.0
#42 0x00007ffff67ef2ba in ??? () at /usr/lib/x86_64-linux-gnu/gstreamer-1.0/libgstrtsp.so
#43 0x00007ffff67f8ba0 in ??? () at /usr/lib/x86_64-linux-gnu/gstreamer-1.0/libgstrtsp.so
#44 0x00007ffff67fe585 in ??? () at /usr/lib/x86_64-linux-gnu/gstreamer-1.0/libgstrtsp.so
#45 0x00007ffff7f0abb4 in ??? () at /lib/x86_64-linux-gnu/libgstreamer-1.0.so.0
#46 0x00007ffff7d2a422 in ??? () at /lib/x86_64-linux-gnu/libglib-2.0.so.0
#47 0x00007ffff7d24e62 in ??? () at /lib/x86_64-linux-gnu/libglib-2.0.so.0
#48 0x00007ffff709caa4 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:447
#49 0x00007ffff7129c6c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78

The test I did for container was with the same command. I keep it from the doc.

For the container, the deepstream-server start and work normaly:
root@gpuserver00:/opt/nvidia/deepstream/deepstream-8.0/sources/apps/sample_apps/deepstream-server# ./deepstream-server-app ./dsserver_config.yml
Setting rest server callbacks
Calling nvds_rest_server_start from the server app
Civetweb version: v1.16
Server running at port: 9000
Calling nvmultiurisrcbincreator API
!! [WARNING] “encoder” group not enabled.
Using file: ./dsserver_config.yml
Running…
uri:/api/v1/stream/add
method:POST
Inside s_stream_callback_impl callback +++
stream_info->value_change is stream add
we can accomodate more sources
sensor id don’t exist
Adding source now
Successfully added sensor id=[uniqueSensorID1] uri=[rtsp://192.168.1.113:8554/test]
Exiting s_stream_callback_impl callback

I will waiting for your answer but I would like to reinstall Ubuntu. Thas going to be one week I’m stuck by this issus and it’s the final solution in general. I have all important source on git so it’s not an importante impact.

Did you think it’s a good option to return on a normal situation quickly ?

This stack confirms my suspicion. Try the following two workarounds.

# Set the environment variable before launching the application to prevent GIO from using the libproxy resolver, which avoids the Duktape symbol conflict entirely:
export GIO_USE_PROXY_RESOLVER=dummy

export no_proxy="*"
# Or
apt remove libproxy1v5
# or remove only the PAC/duktape-related libproxy module

Ho it’s work. You are a genius.

So, what it the probleme ?

This is a symbol conflict issue.

libpxbackend uses duktape, and libcivetweb contains the same symbol(from duktape too, and it is load first), but it cannot work with libpxbackend.

Therefore, uninstalling libproxy1v5 to prevent the loading of the duktape symbol should solve the problem, or disabling the proxy should also work.

Ok, it sure I would never have found that myself.

Thank you very much for your help. You have saved my mental health.