Hi, Sorry for reply late.
I’m able to change the frame by applying the demo you offered earlier. But, the change is inside the custom plugin. The next element after gets the original frame, but not the changed frame.
The function below do the job of change frame and it is called in gst_capture_submit_input_buffer() which is simillar to gst_dsexample_submit_input_buffer().
static GstFlowReturn
form_batch_and_push (NvDsBatchMeta* batch_meta, NvBufSurface* in_surf,
GstBuffer* inbuf, GstCapture* capture)
{
guint num_filled = batch_meta->num_frames_in_batch;
auto it = capture->capture_nw->begin();
gdouble scale_ratio = 1.0;
GstFlowReturn ret = GST_FLOW_OK;
std::unique_ptr <GstCaptureBatch> batch = nullptr;
int sync_device_result = 0;
cv::Mat tmp, nv12_after;
for (guint i = 0; i < num_filled && it != capture->capture_nw->end(); i++, it++)
{
g_print (" batch push: %u\n", i);
if (batch == nullptr)
{
batch.reset (new GstCaptureBatch);
batch->push_buffer = FALSE;
batch->inbuf = inbuf;
batch->inbuf_batch_num = CAPTURE_BATCH_NUM;
}
guint height = in_surf->surfaceList[i].height;
guint width = in_surf->surfaceList[i].width;
/* Adding a frame to the current batch. Set the frames members. */
GstCaptureFrame frame;
frame.scale_ratio_x = scale_ratio;
frame.scale_ratio_y = scale_ratio;
frame.obj_meta = nullptr;
frame.frame_meta = nvds_get_nth_frame_meta (batch_meta->frame_meta_list, i);
g_print(" form frameNum: %d\n", it->first);
auto saved_mat = capture->track_frame_meta.get(it->first);
if (saved_mat == nullptr)
{
g_print("data is null\n");
ret = GST_FLOW_ERROR;
return ret;
}
// todo: set saved_mat to input_surf_params, do this in modify_output_...
cv::Mat frame_image(height, width, CV_8UC3,
saved_mat);
std::string save_name = "output/save_image_" + std::to_string(it->first) + ".jpg";
imwrite(save_name, frame_image);
//do transform: bgr -> nv12, and cpu_data -> device_data
NvBufSurface* inter_buf = nullptr;
NvBufSurfaceCreateParams create_params;
create_params.gpuId = in_surf->gpuId;
create_params.width = width;
create_params.height = height;
create_params.size = 0;
create_params.colorFormat = NVBUF_COLOR_FORMAT_BGRA;
create_params.layout = NVBUF_LAYOUT_PITCH;
#ifdef __aarch64__
create_params.memType = NVBUF_MEM_DEFAULT;
#else
create_params.memType = NVBUF_MEM_CUDA_UNIFIED;
#endif
//Create another scratch RGBA NvBufSurface
if (NvBufSurfaceCreate (&inter_buf, 1,
&create_params) != 0)
{
GST_ERROR ("Error: Could not allocate internal buffer ");
return GST_FLOW_OK;
}
if (NvBufSurfaceMap (inter_buf, 0, -1, NVBUF_MAP_READ_WRITE) != 0)
g_print("map error\n");
NvBufSurfaceSyncForCpu (inter_buf, 0, 0);
Mat inter_buf_mat = Mat(height, width, CV_8UC4,
inter_buf->surfaceList[0].mappedAddr.addr[0],
inter_buf->surfaceList[0].pitch);
// Aplly your algo which works with opencv Mat, here we only rotate the Mat for demo
// rotate(rgba_mat, rotate_mat, ROTATE_180);
rectangle (frame_image, Point(100, 300), Point(700, 500), Scalar(0, 255, 255), 2, 8, 0);
cvtColor(frame_image, inter_buf_mat, COLOR_BGR2BGRA);
sync_device_result = NvBufSurfaceSyncForDevice(inter_buf, 0, 0);
g_print("sync_device_result inter_buf: %d\n", sync_device_result);
inter_buf->numFilled = 1;
NvBufSurfTransformConfigParams transform_config_params;
NvBufSurfTransformParams transform_params;
NvBufSurfTransformRect src_rect;
NvBufSurfTransformRect dst_rect;
cudaStream_t cuda_stream;
CHECK_CUDA_STATUS (cudaStreamCreate (&cuda_stream),
"Could not create cuda stream");
transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
transform_config_params.gpu_id = in_surf->gpuId;
transform_config_params.cuda_stream = cuda_stream;
/* Set the transform session parameters for the conversions executed in this
* thread. */
NvBufSurfTransform_Error err = NvBufSurfTransformSetSessionParams (&transform_config_params);
if (err != NvBufSurfTransformError_Success)
{
g_print("NvBufSurfTransformSetSessionParams failed with error %d\n", err);
return GST_FLOW_OK;
}
/* Set the transform ROIs for source and destination, only do the color format conversion*/
src_rect = {0, 0, width, height};
dst_rect = {0, 0, width, height};
/* Set the transform parameters */
transform_params.src_rect = &src_rect;
transform_params.dst_rect = &dst_rect;
transform_params.transform_flag =
NVBUFSURF_TRANSFORM_FILTER | NVBUFSURF_TRANSFORM_CROP_SRC |
NVBUFSURF_TRANSFORM_CROP_DST;
transform_params.transform_filter = NvBufSurfTransformInter_Default;
/* Transformation format conversion, Transform rotated RGBA mat to NV12 memory in original input surface*/
err = NvBufSurfTransform (inter_buf, in_surf, &transform_params);
if (err != NvBufSurfTransformError_Success)
{
g_print("NvBufSurfTransform failed with error %d while converting buffer %d\n", err);
return GST_FLOW_OK;
}
sync_device_result = NvBufSurfaceSyncForDevice(in_surf, 0, 0);
g_print("sync_device_result in_buf: %d\n", sync_device_result);
NvBufSurfaceUnMap(inter_buf, 0, 0);
tmp = cv::Mat(height * 3 / 2, width, CV_8UC1, in_surf->surfaceList[i].mappedAddr.addr[0],
in_surf->surfaceList[i].pitch);
cv::cvtColor(tmp, nv12_after, CV_YUV2BGRA_NV12);
std::string file_name = "output/nv12_after_" + std::to_string(it->first) + ".jpg";
imwrite(file_name, nv12_after);
// endtodo: set saved_mat to input_surf_params
frame.frame_num = it->first;
frame.batch_index = i;
frame.input_surf_params = in_surf->surfaceList + i;
batch->frames.push_back (frame);
// Set the transform session parameters for the conversions executed in this
// thread.
if (batch->frames.size () == capture->max_batch_size
|| i == batch_meta->num_frames_in_batch)
{
// print("point 5\n");
g_mutex_lock (&capture->process_lock);
g_queue_push_tail (capture->process_queue, batch.get());
g_cond_broadcast (&capture->process_cond);
g_mutex_unlock (&capture->process_lock);
/* Batch submitted. Set batch to nullptr so that a new GstCaptureBatch
* structure can be allocated if required. */
batch.release ();
}
if (ret == GST_FLOW_ERROR)
{
g_print ("error occur 2\n");
}
}
return ret;
error:
return GST_FLOW_ERROR;
}