I featched image and use opencv to do the image process, but I think using NPP API to do image process is the better way(because it on GPU but we dont have experience on this API)
there are two way to get image from gstBuffer:
1.using cudaMemcpy:
code piece as below:
GstBuffer *buf = (GstBuffer *) info->data;
guint num_rects = 0;
NvDsObjectMeta *obj_meta = NULL;
guint vehicle_count = 0;
guint person_count = 0;
NvDsMetaList * l_frame = NULL;
NvDsMetaList * l_obj = NULL;
GstMapInfo in_map_info;
if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
g_print ("Error: Failed to map gst buffer\n");
gst_buffer_unmap (buf, &in_map_info);
return GST_PAD_PROBE_OK;
}
std::vector<cv::Mat>frames;
NvBufSurface *surface = (NvBufSurface *)in_map_info.data;
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
l_frame = l_frame->next) {
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
NvBufSurfaceParams frameHandle=surface->surfaceList[frame_meta->batch_id];
cv::Mat rawFrame;
bool color_format_supported=true;
switch(frameHandle.colorFormat){
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_RGBA:
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_RGBx:
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_BGRA:
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_BGRx:
rawFrame.create(cv::Size(frameHandle.width,frameHandle.height),CV_8UC4);
break;
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_GRAY8:
rawFrame.create(cv::Size(frameHandle.width,frameHandle.height),CV_8UC1);
break;
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_BGR:
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_RGB:
rawFrame.create(cv::Size(frameHandle.width,frameHandle.height),CV_8UC3);
break;
default:{
g_print("unsupport color format tosave :%d in source:%d frame:%d",frameHandle.colorFormat,frame_meta->source_id,frame_meta->frame_num);
color_format_supported=false;
}
}
if(color_format_supported){
HANDLE_ERROR(cudaMemcpy(rawFrame.data,frameHandle.dataPtr,frameHandle.dataSize,cudaMemcpyDeviceToHost));
switch(frameHandle.colorFormat){
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_RGBA:
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_RGBx:
cv::cvtColor(rawFrame,rawFrame,CV_RGBA2BGR);
break;
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_BGRA:
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_BGRx:
cv::cvtColor(rawFrame,rawFrame,CV_BGRA2BGR);
break;
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_GRAY8:
cv::cvtColor(rawFrame,rawFrame,CV_GRAY2BGR);
break;
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_BGR:
break;
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_RGB:
cv::cvtColor(rawFrame,rawFrame,CV_RGB2BGR);
break;
default:
g_print("unsupport color format tosave :%d in source:%d frame:%d",frameHandle.colorFormat,frame_meta->source_id,frame_meta->frame_num);
color_format_supported=false;
}
}
PS: dont forget call gst_buffer_unmap (buf, &in_map_info); before return
-
using the global virual memory access (this concept mentioned in cuda.)
first need set the property for streammux and nvvideoconvert:
then you can do this to get the image data as cv::Mat:
bool GetCVMatFromNvBuf(GstBuffer * buf,std::vectorcv::Mat&frames){
frames.clear();
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
frames=std::vectorcv::Mat( batch_meta->num_frames_in_batch);
GstMapInfo in_map_info;
//get surface
if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
g_print ("Error: Failed to map gst buffer\n");
gst_buffer_unmap (buf, &in_map_info);
return false;
}
NvBufSurface *surface = (NvBufSurface *)in_map_info.data;
NvDsMetaList* l_frame;
NvBufSurfaceMap (surface, -1, -1, NVBUF_MAP_READ);
NvBufSurfaceSyncForCpu (surface, 0, 0);
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;l_frame = l_frame->next) {
cv::Mat tmp;
NvDsFrameMeta* frameInfo=( NvDsFrameMeta*)l_frame->data;
int height=surface->surfaceList[frameInfo->batch_id].height;
int width=surface->surfaceList[frameInfo->batch_id].width;
switch(surface->surfaceList[frameInfo->batch_id].colorFormat){
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_BGRA:
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_BGRx:
tmp=cv::Mat(height,width,CV_8UC4,surface->surfaceList[frameInfo->batch_id].mappedAddr.addr[0],surface->surfaceList[frameInfo->batch_id].pitch);
cv::cvtColor(tmp,frames[frameInfo->batch_id],CV_BGRA2BGR);
break;
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_RGBA:
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_RGBx:
tmp=cv::Mat(height,width,CV_8UC4,surface->surfaceList[frameInfo->batch_id].mappedAddr.addr[0],surface->surfaceList[frameInfo->batch_id].pitch);
cv::cvtColor(tmp,frames[frameInfo->batch_id],CV_RGBA2BGR);
break;
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_RGB:
tmp=cv::Mat(height,width,CV_8UC3,surface->surfaceList[frameInfo->batch_id].mappedAddr.addr[0],surface->surfaceList[frameInfo->batch_id].pitch);
cv::cvtColor(tmp,frames[frameInfo->batch_id],CV_RGB2BGR);
break;
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_BGR:
tmp=cv::Mat(height,width,CV_8UC3,surface->surfaceList[frameInfo->batch_id].mappedAddr.addr[0],surface->surfaceList[frameInfo->batch_id].pitch);
frames[frameInfo->batch_id]=tmp.clone();
break;
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_GRAY8:
tmp=cv::Mat(height,width,CV_8UC1,surface->surfaceList[frameInfo->batch_id].mappedAddr.addr[0],surface->surfaceList[frameInfo->batch_id].pitch);
cv::cvtColor(tmp,frames[frameInfo->batch_id],CV_GRAY2BGR);
break;
case NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_NV12:
tmp=cv::Mat(height*3/2,width,CV_8UC1,surface->surfaceList[frameInfo->batch_id].mappedAddr.addr[0],surface->surfaceList[frameInfo->batch_id].pitch);
cv::cvtColor(tmp,frames[frameInfo->batch_id],CV_YUV2BGR_NV12);
break;
default:
g_print("unsupport color format tosave :%d in source:%d frame:%d",surface->surfaceList[frameInfo->batch_id].colorFormat,frameInfo->source_id,frameInfo->frame_num);
frames.clear();
return false;
}
}
gst_buffer_unmap (buf, &in_map_info);
return true;
}
for more about gistBuffer, NvBufSurface,GstNvInferBatch, i think you need to check deepstream api doc and gstreamer doc.
may it can help u. ;)