Ciao Fiona,
Good morning :)
Please find here below the results of the tests:
-
Original test3 app: it works perfectly
-
Test3 app + gst-dsexample in the pipeline to save frames after PGIE: it works perfectly
-
Test3 app + gst-dsexample in the pipeline to save frames after PGIE + pre-processing with gst-dsexample to all frames before PGIE (implementation as in lines 1170 - 1240 of gstdsexample.cpp - codes follow below) : segmentation fault
} else if (0) { //if (1) { NvDsMetaList * l_obj = NULL; for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) { frame_meta = (NvDsFrameMeta *) (l_frame->data); cv::Mat in_mat, conv_mat; NvOSD_RectParams rect_params; /* Scale the entire frame to processing resolution */ rect_params.left = 0; rect_params.top = 0; rect_params.width = dsexample->video_info.width; rect_params.height = dsexample->video_info.height; l_obj = frame_meta->obj_meta_list; if (1) { /* Map the buffer so that it can be accessed by CPU */ if (surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0] == NULL){ if (NvBufSurfaceMap (surface, frame_meta->batch_id, 0, NVBUF_MAP_READ_WRITE) != 0){ GST_ELEMENT_ERROR (dsexample, STREAM, FAILED, ("%s:buffer map to be accessed by CPU failed", __func__), (NULL)); return GST_FLOW_ERROR; } } /* Cache the mapped data for CPU access */ NvBufSurfaceSyncForCpu (surface, frame_meta->batch_id, 0); in_mat = cv::Mat (surface->surfaceList[frame_meta->batch_id].planeParams.height[0], surface->surfaceList[frame_meta->batch_id].planeParams.width[0], CV_8UC4, surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0], surface->surfaceList[frame_meta->batch_id].planeParams.pitch[0]); } //cv::cvtColor (in_mat, in_mat, cv::COLOR_RGBA2BGR); //cv::filter2D(in_mat, in_mat,-1, kernel); //cv::Mat image_copy = in_mat.clone(); //cv::cvtColor (image_copy, image_copy, cv::COLOR_RGBA2BGR); //g_print ("Transform start"); cv::transform(in_mat, in_mat, kernel); //g_print ("Transform done"); //cv::cvtColor (in_mat, in_mat, cv::COLOR_BGR2RGBA); //in_mat.convertTo(in_mat,CV_8UC4); //g_print("\n inmat \n"); //g_print(in_mat); //std::cout << "conv_mat = " << std::endl << " " << image_copy << std::endl << std::endl; //int from_to[] = { 0,0, 1,1, 2,2, 3,3 }; //cv::mixChannels( &in_mat, 1, &in_mat, 1, from_to, 4 ); /* Process the object crop to obtain label */ //output = DsExampleProcess (dsexample->dsexamplelib_ctx, dsexample->cvmat->data); /* Attach labels for the object */ //attach_metadata_object (dsexample, obj_meta, output); //free (output); //cv::cvtColor (in_mat, in_mat, cv::COLOR_RGBA2BGRA); //g_print(std::to_string(frame_meta->batch_id)); //std::cout << surface->surfaceList[frame_meta->batch_id].planeParams.pitch[0] << std::endl; NvBufSurfaceSyncForDevice (surface, frame_meta->batch_id, 0); } -
Test3 app + gst-dsexample in the pipeline to save frames after PGIE + pre-processing with gst-dsexample to all frames before PGIE (implementation as in lines 1136- 1169 of gstdsexample.cpp - codes follow below): no segmentation fault, but the image that arrives to PGIE is NOT processed. It seems that I am not doing anything to the buffer
} else if (!dsexample->process_full_frame) { //if (1) { NvDsMetaList * l_obj = NULL; for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) { frame_meta = (NvDsFrameMeta *) (l_frame->data); NvOSD_RectParams rect_params; /* Scale the entire frame to processing resolution */ rect_params.left = 0; rect_params.top = 0; rect_params.width = dsexample->video_info.width; rect_params.height = dsexample->video_info.height; l_obj = frame_meta->obj_meta_list; if (frame_meta->obj_meta_list != NULL){ if (filter_frame (dsexample, surface, i, &rect_params, scale_ratio, dsexample->video_info.width, dsexample->video_info.height) != GST_FLOW_OK) { goto error; } } /* Process to get the output */ output = DsExampleProcess (dsexample->dsexamplelib_ctx, dsexample->cvmat->data); /* Attach the metadata for the full frame */ //attach_metadata_full_frame (dsexample, frame_meta, scale_ratio, output, i); i++; free (output); }
The function filter_frame is a modified version of get_converted_mat:
static GstFlowReturn
filter_frame (GstDsExample * dsexample, NvBufSurface *input_buf, gint idx,
NvOSD_RectParams * crop_rect_params, gdouble & ratio, gint input_width,
gint input_height)
{
NvBufSurfTransform_Error err;
NvBufSurfTransformConfigParams transform_config_params;
NvBufSurfTransformParams transform_params;
NvBufSurfTransformRect src_rect;
NvBufSurfTransformRect dst_rect;
NvBufSurface ip_surf;
cv::Mat in_mat, out_mat, filtered_mat;
ip_surf = *input_buf;
time_t rawtime;
struct tm *info;
ip_surf.numFilled = ip_surf.batchSize = 1;
ip_surf.surfaceList = &(input_buf->surfaceList[idx]);
/*
gint src_left = GST_ROUND_UP_2(crop_rect_params->left);
gint src_top = GST_ROUND_UP_2(crop_rect_params->top);
gint src_width = GST_ROUND_DOWN_2(crop_rect_params->width);
gint src_height = GST_ROUND_DOWN_2(crop_rect_params->height);
*/
gint src_left = crop_rect_params->left;
gint src_top = crop_rect_params->top;
gint src_width = crop_rect_params->width;
gint src_height = crop_rect_params->height;
//g_print("ltwh = %d %d %d %d \n", src_left, src_top, src_width, src_height);
guint dest_width, dest_height;
dest_width = src_width;
dest_height = src_height;
NvBufSurface *nvbuf;
NvBufSurfaceCreateParams create_params;
create_params.gpuId = dsexample->gpu_id;
create_params.width = dest_width;
create_params.height = dest_height;
create_params.size = 0;
create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
create_params.layout = NVBUF_LAYOUT_PITCH;
#ifdef __aarch64__
create_params.memType = NVBUF_MEM_DEFAULT;
#else
create_params.memType = NVBUF_MEM_CUDA_UNIFIED;
#endif
NvBufSurfaceCreate (&nvbuf, 1, &create_params);
// Configure transform session parameters for the transformation
transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
transform_config_params.gpu_id = dsexample->gpu_id;
transform_config_params.cuda_stream = dsexample->cuda_stream;
// Set the transform session parameters for the conversions executed in this
// thread.
err = NvBufSurfTransformSetSessionParams (&transform_config_params);
if (err != NvBufSurfTransformError_Success) {
GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
("NvBufSurfTransformSetSessionParams failed with error %d", err), (NULL));
goto error;
}
// Calculate scaling ratio while maintaining aspect ratio
ratio = MIN (1.0 * dest_width/ src_width, 1.0 * dest_height / src_height);
if ((crop_rect_params->width == 0) || (crop_rect_params->height == 0)) {
GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
("%s:crop_rect_params dimensions are zero",__func__), (NULL));
goto error;
}
#ifdef __aarch64__
if (ratio <= 1.0 / 16 || ratio >= 16.0) {
// Currently cannot scale by ratio > 16 or < 1/16 for Jetson
goto error;
}
#endif
// Set the transform ROIs for source and destination
src_rect = {(guint)src_top, (guint)src_left, (guint)src_width, (guint)src_height};
dst_rect = {0, 0, (guint)dest_width, (guint)dest_height};
// Set the transform parameters
transform_params.src_rect = &src_rect;
transform_params.dst_rect = &dst_rect;
transform_params.transform_flag =
NVBUFSURF_TRANSFORM_FILTER | NVBUFSURF_TRANSFORM_CROP_SRC |
NVBUFSURF_TRANSFORM_CROP_DST;
transform_params.transform_filter = NvBufSurfTransformInter_Default;
//Memset the memory
NvBufSurfaceMemSet (nvbuf, 0, 0, 0);
GST_DEBUG_OBJECT (dsexample, "Scaling and converting input buffer\n");
// Transformation scaling+format conversion if any.
err = NvBufSurfTransform (&ip_surf, nvbuf, &transform_params);
if (err != NvBufSurfTransformError_Success) {
GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
("NvBufSurfTransform failed with error %d while converting buffer", err),
(NULL));
goto error;
}
// Map the buffer so that it can be accessed by CPU
//if (NvBufSurfaceMap (nvbuf, 0, 0, NVBUF_MAP_READ) != 0){
if (NvBufSurfaceMap (nvbuf, 0, 0, NVBUF_MAP_READ_WRITE) != 0){
goto error;
}
// Cache the mapped data for CPU access
NvBufSurfaceSyncForCpu (nvbuf, 0, 0);
// Use openCV to remove padding and convert RGBA to BGR. Can be skipped if
// algorithm can handle padded RGBA data.
in_mat =
cv::Mat (dest_height, dest_width,
CV_8UC4, nvbuf->surfaceList[0].mappedAddr.addr[0],
nvbuf->surfaceList[0].pitch);
out_mat =
cv::Mat (cv::Size(dest_width, dest_height), CV_8UC3);
filtered_mat =
cv::Mat (cv::Size(dest_width, dest_height), CV_8UC3);
//cv::cvtColor (in_mat, out_mat, cv::COLOR_RGBA2BGR);
//cv::transform(out_mat, filtered_mat, kernel);
//cv::cvtColor (in_mat, in_mat, cv::COLOR_RGBA2BGR);
cv::transform(in_mat, in_mat, kernel);
//cv::cvtColor (in_mat, in_mat, cv::COLOR_BGR2RGBA);
/* Cache the mapped data for device access */
NvBufSurfaceSyncForDevice (nvbuf, 0, 0);
/*
time( &rawtime );
info = localtime( &rawtime );
static gint dump = 0;
char filename[64];
snprintf(filename, 64, "/home/jnano/jnanoImages/%04d_%02d_%02d_%02d_%02d_%02d.jpg", info->tm_year+1900, info->tm_mon+1, info->tm_mday+1, info->tm_hour, info->tm_min, info->tm_sec);
cv::imwrite(filename, saved_mat);
*/
if (NvBufSurfaceUnMap (nvbuf, 0, 0)){
goto error;
}
NvBufSurfaceDestroy(nvbuf);
#ifdef __aarch64__
// To use the converted buffer in CUDA, create an EGLImage and then use
// CUDA-EGL interop APIs
if (USE_EGLIMAGE) {
if (NvBufSurfaceMapEglImage (dsexample->inter_buf, 0) !=0 ) {
goto error;
}
// dsexample->inter_buf->surfaceList[0].mappedAddr.eglImage
// Use interop APIs cuGraphicsEGLRegisterImage and
// cuGraphicsResourceGetMappedEglFrame to access the buffer in CUDA
// Destroy the EGLImage
NvBufSurfaceUnMapEglImage (dsexample->inter_buf, 0);
}
#endif
/* We will first convert only the Region of Interest (the entire frame or the
* object bounding box) to RGB and then scale the converted RGB frame to
* processing resolution. */
return GST_FLOW_OK;
error:
return GST_FLOW_ERROR;
}
I really hope that you can help me @Fiona.Chen .
Thank you very much in advance and have a nice day!