• Hardware Platform (Jetson / GPU) RTX 3090
• DeepStream Version 6.3
• TensorRT Version 8.5.3
• NVIDIA GPU Driver Version (valid for GPU only) 535.129.03
• Issue Type( questions, new requirements, bugs) questions
I need to crop detections from frames. My pipeline is following:
streammux -> pgie -> nvtracker -> nvvidconv -> nvvidconv_cap -> fake_sink
where nvvidconv and nvvidconv_cap are set for converting NVMM to RGB:
GstElement *nvvidconv = gst_element_factory_make("nvvideoconvert", "nvvideo-converter");
g_object_set (G_OBJECT (nvvidconv), "nvbuf-memory-type", 3, NULL);
GstElement *nvvidconv_cap = gst_element_factory_make("capsfilter", "nvvidconv_cap");
GstCaps *caps = gst_caps_from_string ("video/x-raw(memory:NVMM),format=RGB");
g_object_set (G_OBJECT (nvvidconv_cap), "caps", caps, NULL);
To crop the detection from the frame I use NvBufSurfTransform function. For batchsize = 1, I create new NvBufSurface *dst_buf for every detection and this way I can specify the size of the NvBufSurface *dst_buf as crop detection size:
#include <glib.h>
#include <gst/gst.h>
#include "gstnvdsmeta.h"
#include "nvbufsurface.h"
#include "nvds_obj_encode.h"
#include "nvbufsurftransform.h"
GstPadProbeReturn metadata_probe(GstPad *pad, GstPadProbeInfo *info, gpointer u_data)
{
/* Get buffer. */
GstBuffer *buf = (GstBuffer *)info->data;
GstMapInfo inmap = GST_MAP_INFO_INIT;
if (!gst_buffer_map(buf, &inmap, GST_MAP_READ))
{
glog(__FILE__, G_LOG_LEVEL_WARNING, "Input buffer mapinfo failed\n");
return GST_FLOW_ERROR;
}
NvBufSurface *ip_surf = (NvBufSurface *)inmap.data;
gst_buffer_unmap(buf, &inmap);
// Set the transform configuration parameters
NvBufSurfTransformConfigParams transform_config_params;
NvBufSurfTransform_Error err;
transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
transform_config_params.gpu_id = 0;
transform_config_params.cuda_stream = 0;
NvBufSurfTransformSetSessionParams(&transform_config_params);
NvBufSurface *dst_buf;
NvBufSurfaceCreateParams create_params;
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf);
NvDsMetaList *l_frame = NULL;
NvDsObjectMeta *obj_meta = NULL;
NvDsMetaList *l_obj = NULL;
gboolean medata_was_sent = FALSE;
/* Go throught all frames in the batch. */
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next)
{
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data);
if (frame_meta->num_obj_meta > 0)
{
guint source_id = frame_meta->source_id;
/* Go throught all detected objects in the frame. */
for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next)
{
obj_meta = (NvDsObjectMeta *)(l_obj->data);
NvBbox_Coords *bbox_coords = &obj_meta->detector_bbox_info.org_bbox_coords;
NvBufSurfTransformRect src_rect;
src_rect.top = bbox_coords->top;
src_rect.left = bbox_coords->left;
src_rect.width = bbox_coords->width;
src_rect.height = bbox_coords->height;
// Set the destination rectangle (usually same as crop size)
NvBufSurfTransformRect dst_rect;
dst_rect.top = 0;
dst_rect.left = 0;
dst_rect.width = src_rect.width;
dst_rect.height = src_rect.height;
// Transformation parameters
NvBufSurfTransformParams transform_params;
transform_params.src_rect = &src_rect;
transform_params.dst_rect = &dst_rect;
transform_params.transform_flag = NVBUFSURF_TRANSFORM_CROP_SRC | NVBUFSURF_TRANSFORM_CROP_DST;
transform_params.transform_filter = NvBufSurfTransformInter_Default;
/* Create dst buffer. */
NvBufSurface *dst_buf;
NvBufSurfaceCreateParams create_params;
// Set the parameters for the buffer
create_params.gpuId = 0;
create_params.width = bbox_coords->width; // Set the width of the crop region
create_params.height = bbox_coords->height; // Set the height of the crop region
create_params.size = 0;
create_params.layout = NVBUF_LAYOUT_PITCH; // Choose the appropriate layout
create_params.colorFormat = NVBUF_COLOR_FORMAT_RGB; // Choose the appropriate color format
create_params.memType = ip_surf->memType; // Memory type, e.g., device memory
// Allocate the buffer
if (NvBufSurfaceCreate(&dst_buf, 1, &create_params) != 0)
{
// Handle error
g_print("Error: NvBufSurfaceCreate has failed.\n");
}
NvBufSurfaceMemSet(dst_buf, 0, 0, 0);
NvBufSurfaceMap(dst_buf, 0, 0, NVBUF_MAP_READ);
NvBufSurfaceSyncForCpu(dst_buf, 0, 0);
// Perform the transformation (crop)
err = NvBufSurfTransform(ip_surf, dst_buf, &transform_params);
if (err != NvBufSurfTransformError_Success)
{
g_print("NvBufSurfTransform has failed: %d\n", err);
}
uint32_t Y_stride = dst_buf->surfaceList[0].planeParams.pitch[0];
const uint8_t *Y = dst_buf->surfaceList[0].dataPtr;
}
}
}
return GST_PAD_PROBE_OK;
}
Where const uint8_t *Y is a pointer to the crop image. This metadata_probe function is added to the src pad of nvvidconv_cap. This solution works for me and I can save and display the crop images.
But for the case when batchsize > 1, I am not sure how to work with that. I can not create new NvBufSurface *dst_buf for every detection because src and dst buffers in NvBufSurfTransform function must have the same batchsize (as I know). My main confusions are:
- How to set the size of destination
NvBufSurface(create_params.width and create_params.height or create_params.size )? In every frame in batch, there can be many detection with different sizes. - How can I differentiate detections between frames in batch for src_rect and dst_rect arrays in
NvBufSurfTransformParams? How does NvBufSurfTransform function know, which detection belongs to which frame? - How to get all detection crops from the dst buffer?