I am working with deepstream image meta-test example. I understood the process of saving a full-frame image. But How can I add the detected object to that frame? Please help me to add a bounding box and save a full-frame image.
GPU Type: RTX 3070Ti
Nvidia Driver Version: 535.146.02
CUDA Version: 12.2
Operating System + Version: Ubuntu 22.04
Please add the probe function to the src pad of the osd.
osd_src_pad = gst_element_get_static_pad (nvosd, "src");
if (!osd_src_pad )
g_print ("Unable to get src pad\n");
else
gst_pad_add_probe (osd_src_pad , GST_PAD_PROBE_TYPE_BUFFER,
osd_sink_pad_buffer_probe, (gpointer) obj_ctx_handle, NULL);
gst_object_unref (osd_src_pad );
Please set the frameData.saveImg to True.
Thank you for replying @yuweiw
I added whichever you mentioned. But there are no bounding boxes in full-frame images. I am attaching the c file. Please go through it.
static GstPadProbeReturn
osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info,
gpointer u_data)
{
GstBuffer *buf = (GstBuffer *) info->data;
guint num_rects = 0;
NvDsObjectMeta *obj_meta = NULL;
guint vehicle_count = 0;
guint person_count = 0;
NvDsMetaList *l_frame = NULL;
NvDsMetaList *l_obj = NULL;
NvDsDisplayMeta *display_meta = NULL;
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
NvOSD_RectParams *rect_params = display_meta->rect_params;
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
l_frame = l_frame->next) {
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
int offset = 0;
/* To verify encoded metadata of cropped frames, we iterate through the
* user metadata of each frame and if a metadata of the type
* 'NVDS_CROP_IMAGE_META' is found then we write that to a file as
* implemented below.
*/
char fileFrameNameString[FILE_NAME_SIZE];
const char *osd_string = "OSD";
/* For Demonstration Purposes we are writing metadata to jpeg images of
* the first 10 frames only.
* The files generated have an 'OSD' prefix. */
if (frame_number < 11) {
NvDsUserMetaList *usrMetaList = frame_meta->frame_user_meta_list;
FILE *file;
int stream_num = 0;
while (usrMetaList != NULL) {
NvDsUserMeta *usrMetaData = (NvDsUserMeta *) usrMetaList->data;
if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
// snprintf (fileFrameNameString, FILE_NAME_SIZE, "%s_frame_%d_%d.jpg",
// osd_string, frame_number, stream_num++);
snprintf (fileFrameNameString, FILE_NAME_SIZE, "new/%s_frame_%d_%d.jpg",osd_string, frame_number, stream_num++);
NvDsObjEncOutParams *enc_jpeg_image = (NvDsObjEncOutParams *) usrMetaData->user_meta_data;
/* Write to File */
file = fopen (fileFrameNameString, "wb");
fwrite (enc_jpeg_image->outBuffer, sizeof (uint8_t),enc_jpeg_image->outLen, file);
fclose (file);
}
usrMetaList = usrMetaList->next;
}
}
for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) {
obj_meta = (NvDsObjectMeta *) (l_obj->data);
if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) {
vehicle_count++;
num_rects++;
}
if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) {
person_count++;
num_rects++;
}
/* To verify encoded metadata of cropped objects, we iterate through the
* user metadata of each object and if a metadata of the type
* 'NVDS_CROP_IMAGE_META' is found then we write that to a file as
* implemented below.
*/
char fileObjNameString[FILE_NAME_SIZE];
/* For Demonstration Purposes we are writing metadata to jpeg images of
* vehicles or persons for the first 100 frames only.
* The files generated have a 'OSD' prefix. */
if (frame_number < 100 && (obj_meta->class_id == PGIE_CLASS_ID_PERSON
|| obj_meta->class_id == PGIE_CLASS_ID_VEHICLE)) {
NvDsUserMetaList *usrMetaList = obj_meta->obj_user_meta_list;
FILE *file;
while (usrMetaList != NULL) {
NvDsUserMeta *usrMetaData = (NvDsUserMeta *) usrMetaList->data;
if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
NvDsObjEncOutParams *enc_jpeg_image =
(NvDsObjEncOutParams *) usrMetaData->user_meta_data;
snprintf (fileObjNameString, FILE_NAME_SIZE, "new/%s_%d_%d_%d_%s.jpg", osd_string, frame_number, frame_meta->batch_id, num_rects,
obj_meta->obj_label);
/* Write to File */
file = fopen (fileObjNameString, "wb");
fwrite (enc_jpeg_image->outBuffer, sizeof (uint8_t),
enc_jpeg_image->outLen, file);
fclose (file);
usrMetaList = NULL;
} else {
usrMetaList = usrMetaList->next;
}
}
}
}
display_meta = nvds_acquire_display_meta_from_pool (batch_meta);
NvOSD_TextParams *txt_params = &display_meta->text_params[0];
txt_params->display_text = g_malloc0 (MAX_DISPLAY_LEN);
offset =
snprintf (txt_params->display_text, MAX_DISPLAY_LEN, "Person = %d ",
person_count);
offset =
snprintf (txt_params->display_text + offset, MAX_DISPLAY_LEN,
"Vehicle = %d ", vehicle_count);
/* Now set the offsets where the string should appear */
txt_params->x_offset = 10;
txt_params->y_offset = 12;
/* Font , font-color and font-size */
txt_params->font_params.font_name = "Serif";
txt_params->font_params.font_size = 10;
txt_params->font_params.font_color.red = 1.0;
txt_params->font_params.font_color.green = 1.0;
txt_params->font_params.font_color.blue = 1.0;
txt_params->font_params.font_color.alpha = 1.0;
/* Text background color */
txt_params->set_bg_clr = 1;
txt_params->text_bg_clr.red = 0.0;
txt_params->text_bg_clr.green = 0.0;
txt_params->text_bg_clr.blue = 0.0;
txt_params->text_bg_clr.alpha = 1.0;
nvds_add_display_meta_to_frame (frame_meta, display_meta);
}
g_print ("Frame Number = %d Number of objects = %d "
"Vehicle Count = %d Person Count = %d\n",
frame_number, num_rects, vehicle_count, person_count);
frame_number++;
return GST_PAD_PROBE_OK;
}
static GstPadProbeReturn
pgie_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer ctx)
{
GstBuffer *buf = (GstBuffer *) info->data;
GstMapInfo inmap = GST_MAP_INFO_INIT;
if (!gst_buffer_map (buf, &inmap, GST_MAP_READ)) {
GST_ERROR ("input buffer mapinfo failed");
return GST_PAD_PROBE_DROP;
}
NvBufSurface *ip_surf = (NvBufSurface *) inmap.data;
gst_buffer_unmap (buf, &inmap);
NvDsObjectMeta *obj_meta = NULL;
guint vehicle_count = 0;
guint person_count = 0;
NvDsMetaList *l_frame = NULL;
NvDsMetaList *l_obj = NULL;
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
const gchar *calc_enc_str = g_getenv ("CALCULATE_ENCODE_TIME");
gboolean calc_enc = !g_strcmp0 (calc_enc_str, "yes");
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
l_frame = l_frame->next) {
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
/* For demonstration purposes, we will encode the first 10 frames. */
if(frame_count <= 10) {
NvDsObjEncUsrArgs frameData = { 0 };
/* Preset */
frameData.isFrame = 1;
/* To be set by user */
frameData.saveImg = TRUE;
frameData.attachUsrMeta = attach_user_meta;
/* Set if Image scaling Required */
frameData.scaleImg = FALSE;
frameData.scaledWidth = 0;
frameData.scaledHeight = 0;
/* Quality */
frameData.quality = 80;
/* Set to calculate time taken to encode JPG image. */
if (calc_enc) {
frameData.calcEncodeTime = 1;
}
/* Main Function Call */
nvds_obj_enc_process (ctx, &frameData, ip_surf, NULL, frame_meta);
}
guint num_rects = 0;
for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) {
obj_meta = (NvDsObjectMeta *) (l_obj->data);
if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) {
vehicle_count++;
num_rects++;
}
if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) {
person_count++;
num_rects++;
}
/* Conditions that user needs to set to encode the detected objects of
* interest. Here, by default all the detected objects are encoded.
* For demonstration, we will encode the first object in the frame. */
if ((obj_meta->class_id == PGIE_CLASS_ID_PERSON
|| obj_meta->class_id == PGIE_CLASS_ID_VEHICLE)
&& num_rects == 1) {
NvDsObjEncUsrArgs objData = { 0 };
/* To be set by user */
objData.saveImg = save_img;
objData.attachUsrMeta = attach_user_meta;
/* Set if Image scaling Required */
objData.scaleImg = FALSE;
objData.scaledWidth = 0;
objData.scaledHeight = 0;
/* Preset */
objData.objNum = num_rects;
/* Quality */
objData.quality = 80;
/* Set to calculate time taken to encode JPG image. */
if (calc_enc) {
objData.calcEncodeTime = 1;
}
/*Main Function Call */
nvds_obj_enc_process (ctx, &objData, ip_surf, obj_meta, frame_meta);
}
}
}
nvds_obj_enc_finish (ctx);
frame_count++;
return GST_PAD_PROBE_OK;
}
**In main Function:**
osd_src_pad = gst_element_get_static_pad (nvosd, "src");
if (!osd_src_pad )
g_print ("Unable to get src pad\n");
else
gst_pad_add_probe (osd_src_pad , GST_PAD_PROBE_TYPE_BUFFER, osd_sink_pad_buffer_probe, (gpointer) obj_ctx_handle, NULL);
gst_object_unref (osd_src_pad );
if (!osd_sink_pad)
g_print ("Unable to get sink pad\n");
else
gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, osd_sink_pad_buffer_probe, (gpointer) obj_ctx_handle, NULL);
gst_object_unref (osd_sink_pad);
if (!pgie_src_pad)
g_print ("Unable to get src pad\n");
else
gst_pad_add_probe (pgie_src_pad, GST_PAD_PROBE_TYPE_BUFFER, pgie_src_pad_buffer_probe, (gpointer) obj_ctx_handle, NULL);
gst_object_unref (pgie_src_pad);
You use the nvds_obj_enc_process API in the probe of pgie, but bbox is drawn in nvdsosd plugin. So you need to move the code to the probe function after the nvdsosd plugin.
I just tried
if (!osd_src_pad )
g_print ("Unable to get src pad\n");
else
gst_pad_add_probe (osd_src_pad , GST_PAD_PROBE_TYPE_BUFFER, pgie_src_pad_buffer_probe, (gpointer) obj_ctx_handle, NULL);
gst_object_unref (osd_src_pad );
The cropped images are saved along with bounding boxes. However, the full-frame image is saved only once with all detected objects with bounding boxes.(frame-0)
I have not changed anything in pgie_src_pad_buffer_probe.
Should I change anything, please share any references so that they help new beginners like me can understand.
Thank You.
OK, you can refer the source code I attached, based on deepstream-image-meta-test.
And we suggest you refer to the Guide to learn some basic knowledge about Gstreamer and DeepStream.
deepstream_image_meta_test.c (16.7 KB)
Thank you for your time @yuweiw. Sorry for the late reply.
The frames are saved with bounding boxes.
Will learn the basics of Deepstream.
Thank You