Please provide complete information as applicable to your setup.
• Hardware Platform: RTX 4070
• DeepStream Version: 7.1
• TensorRT Version: 10.7.0
• NVIDIA GPU Driver Version: 570.133.07
• Issue Type: Questions / Bugs
Hello, I have encountered a curious situation with Deepstream, where the application exits with SEGMENTATION FAULT error after a few minutes of processing. The situation is quite rare, as it does not happen often and only on certain live streams.
I suspect the segmentation fault occurs in the nvv4l2decoder element due to the stack trace from gdb (this has been consistent over 20+ trials)
Thread 47 "dec:src" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fff6d600640 (LWP 1610139)]
0x00007ffff7e79ebc in gst_buffer_copy_into () from /lib/x86_64-linux-gnu/libgstreamer-1.0.so.0
(gdb) bt
#0 0x00007ffff7e79ebc in gst_buffer_copy_into () at /lib/x86_64-linux-gnu/libgstreamer-1.0.so.0
#1 0x00007ffff612e106 in () at /usr/lib/x86_64-linux-gnu/gstreamer-1.0/deepstream/libgstnvvideo4linux2.so
#2 0x00007ffff7ee31d7 in () at /lib/x86_64-linux-gnu/libgstreamer-1.0.so.0
#3 0x00007ffff7d0ea34 in g_thread_pool_thread_proxy (data=<optimized out>) at ../glib/gthreadpool.c:350
#4 0x00007ffff7d07a91 in g_thread_proxy (data=0x7fff10001350) at ../glib/gthread.c:831
#5 0x00007ffff7694ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#6 0x00007ffff7726850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
Another interesting observation is that:
- I have encountered this error only if there is some
NvDsMetaadded before the decoder element. - If there are no
NvDsMetaattached to the buffer before decoder, this does not happen
This is the code I have created to demonstrate the issue (it uses mainly code from the Deepstream examples)
#include <string>
#include <glib.h>
#include <gst/gst.h>
#include <gst/gstbuffer.h>
#include <gstnvdsmeta.h>
#include <iostream>
#include <memory>
NvDsMetaType metaType = nvds_get_user_meta_type("EXAMPLE.PTS_ADJUSTED.META");
#define NVDS_DECODER_GST_META_EXAMPLE (nvds_get_user_meta_type("NVIDIA.DECODER.GST_USER_META"))
gint frame_number = 0;
gint parsed_frame_number = 0;
class PtsAdjustedMeta {
public:
guint i1;
guint i2;
gboolean flag;
};
static void meta_release_func(gpointer data, gpointer user_data) {
PtsAdjustedMeta *decoder_meta = (PtsAdjustedMeta *) data;
if (decoder_meta == nullptr) {
return;
}
g_free(decoder_meta);
}
static gpointer meta_copy_func(gpointer data, gpointer user_data) {
// PtsAdjustedMeta *src_decoder_meta = (PtsAdjustedMeta *) data;
// if (src_decoder_meta == nullptr) {
// return nullptr;
// }
// PtsAdjustedMeta *dst_decoder_meta = new PtsAdjustedMeta(*src_decoder_meta);
// return dst_decoder_meta;
return nullptr;
}
static void BufferAdd(GstBuffer *buf) {
PtsAdjustedMeta *pMeta = (PtsAdjustedMeta *) g_malloc0(sizeof(PtsAdjustedMeta));
NvDsMeta *meta = gst_buffer_add_nvds_meta(buf, pMeta, nullptr, meta_copy_func, meta_release_func);
meta->meta_type = metaType;
meta->gst_to_nvds_meta_transform_func = nullptr;
meta->gst_to_nvds_meta_release_func = nullptr;
}
typedef struct _NvDecoderMeta {
guint frame_type;
guint frame_num;
gboolean dec_err;
} NvDecoderMeta;
/* gst meta copy function set by user */
static gpointer decoder_meta_copy_func(gpointer data, gpointer user_data) {
NvDecoderMeta *src_decoder_meta = (NvDecoderMeta *) data;
NvDecoderMeta *dst_decoder_meta = (NvDecoderMeta *) g_malloc0(sizeof(NvDecoderMeta));
memcpy(dst_decoder_meta, src_decoder_meta, sizeof(NvDecoderMeta));
return (gpointer) dst_decoder_meta;
}
/* gst meta release function set by user */
static void decoder_meta_release_func(gpointer data, gpointer user_data) {
NvDecoderMeta *decoder_meta = (NvDecoderMeta *) data;
if (decoder_meta) {
g_free(decoder_meta);
decoder_meta = NULL;
}
}
/* gst to nvds transform function set by user. "data" holds a pointer to NvDsUserMeta */
static gpointer decoder_gst_to_nvds_meta_transform_func(gpointer data, gpointer user_data) {
NvDsUserMeta *user_meta = (NvDsUserMeta *) data;
NvDecoderMeta *src_decoder_meta = (NvDecoderMeta *) user_meta->user_meta_data;
NvDecoderMeta *dst_decoder_meta = (NvDecoderMeta *) decoder_meta_copy_func(src_decoder_meta, NULL);
return (gpointer) dst_decoder_meta;
}
/* release function set by user to release gst to nvds transformed metadata.
* "data" holds a pointer to NvDsUserMeta */
static void decoder_gst_nvds_meta_release_func(gpointer data, gpointer user_data) {
NvDsUserMeta *user_meta = (NvDsUserMeta *) data;
NvDecoderMeta *decoder_meta = (NvDecoderMeta *) user_meta->user_meta_data;
decoder_meta_release_func(decoder_meta, NULL);
}
static GstPadProbeReturn nvdecoder_src_pad_buffer_probe_basic(GstPad *pad, GstPadProbeInfo *info, gpointer u_data) {
GstBuffer *buf = (GstBuffer *) info->data;
NvDsMeta *meta = NULL;
NvDecoderMeta *decoder_meta = (NvDecoderMeta *) g_malloc0(sizeof(NvDecoderMeta));
if (decoder_meta == NULL) {
exit(-1);
}
/* Add dummy metadata */
decoder_meta->frame_type = (frame_number % 3);
decoder_meta->frame_num = frame_number++;
decoder_meta->dec_err = ((frame_number % 4) / 3);
/* Attach decoder metadata to gst buffer using gst_buffer_add_nvds_meta() */
meta = gst_buffer_add_nvds_meta(buf, decoder_meta, NULL, decoder_meta_copy_func, decoder_meta_release_func);
/* Set metadata type */
meta->meta_type = (GstNvDsMetaType) NVDS_DECODER_GST_META_EXAMPLE;
/* Set transform function to transform decoder metadata from Gst meta to
* nvds meta */
meta->gst_to_nvds_meta_transform_func = decoder_gst_to_nvds_meta_transform_func;
/* Set release function to release the transformed nvds metadata */
meta->gst_to_nvds_meta_release_func = decoder_gst_nvds_meta_release_func;
g_print("GST Dec Meta attached with gst decoder output buffer for Frame_Num = %d\n", decoder_meta->frame_num);
g_print(
"Attached decoder Metadata: frame type = %d, frame_num = %d decode_error_status = %d\n",
decoder_meta->frame_type,
decoder_meta->frame_num,
decoder_meta->dec_err
);
return GST_PAD_PROBE_OK;
}
static GstPadProbeReturn nvdecoder_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, gpointer u_data) {
GstBuffer *buf = (GstBuffer *) info->data;
BufferAdd(buf);
g_print("Added meta: %lu\n", buf->pts);
return GST_PAD_PROBE_OK;
}
int main(int argc, char *argv[]) {
gst_init(&argc, &argv);
std::string uri = "rtsp://192.168.5.10:8554/mystream";
// Construct the pipeline string
std::string pipelineDesc = g_strdup_printf(
"urisourcebin uri=%s ! parsebin name=parser ! identity name=i ! nvv4l2decoder name=dec ! "
"nveglglessink sync=0 ",
uri.c_str()
);
// Parse and create the pipeline
GError *error = nullptr;
GstElement *pipeline = gst_parse_launch(pipelineDesc.c_str(), &error);
GstElement *identity = gst_bin_get_by_name(GST_BIN(pipeline), "i");
GstPad *identitySrc = gst_element_get_static_pad(identity, "src");
gst_pad_add_probe(identitySrc, GST_PAD_PROBE_TYPE_BUFFER, nvdecoder_src_pad_buffer_probe_basic, nullptr, nullptr);
if (not pipeline) {
g_printerr("Failed to create pipeline: %s\n", error->message);
g_clear_error(&error);
return -1;
}
gst_element_set_state(pipeline, GST_STATE_PLAYING);
GstBus *bus = gst_element_get_bus(pipeline);
GstMessage *msg =
gst_bus_timed_pop_filtered(bus, GST_CLOCK_TIME_NONE, (GstMessageType) (GST_MESSAGE_ERROR | GST_MESSAGE_EOS));
// Handle message
if (msg != nullptr) {
GError *err;
gchar *debug_info;
switch (GST_MESSAGE_TYPE(msg)) {
case GST_MESSAGE_ERROR:
gst_message_parse_error(msg, &err, &debug_info);
g_printerr("Error received: %s\nDebug: %s\n", err->message, debug_info);
g_clear_error(&err);
g_free(debug_info);
break;
case GST_MESSAGE_EOS:
g_print("End-Of-Stream reached.\n");
break;
default:
break;
}
gst_message_unref(msg);
}
// Cleanup
gst_object_unref(bus);
gst_element_set_state(pipeline, GST_STATE_NULL);
gst_object_unref(pipeline);
return 0;
}
The problem here is that I am not able to provide an input to replicate this exact behavior, as the problematic video is in fact a live stream. I understand that this is inconvenient, but I was unable to provide any alternative (i.e. recording the video stream using ffmpeg and using the file as the input did not replicate the problem).
However I believe it is worth to provide the code and the stack traces, and some insights about the situations in which it happens, as i believe that even with some corrupted frames (which I expect is the case), the application should raise a warning or an exception and end gracefully, rather than raising a segmentation fault.
If there is any more info (i.e. gdb logs, etc.) I will be very happy to help.
Thanks,
Simon