So here’s what I have that appears to work:
tl::expected<cv::Mat, GstFlowReturn> get_opencv_mat(NvBufSurface& ip_surf, gint idx,
gdouble& ratio, gint processing_width,
gint processing_height) {
// Not sure we actually need to copy the NvBufSurface arg above (originally named input_buf), so
// instead of making a copy make it a reference and change the name to ip_surf so it can be used
// directly?
// then we don't need
// NvBufSurface ip_surf;
// ip_surf = *input_buf;
// ip_surf.numFilled = ip_surf.batchSize = 1;
// ip_surf.surfaceList = &(input_buf->surfaceList[idx]);
// Where we're grabbing the entire image, I'm not sure any of this scaling stuff is needed from
// here to the NvBufSurfTransformParams definition
NvOSD_RectParams rect_params;
// Scale the entire frame to processing resolution
rect_params.left = 0;
rect_params.top = 0;
rect_params.width = processing_width;
rect_params.height = processing_height;
gint src_left = GST_ROUND_UP_2((int)rect_params.left);
gint src_top = GST_ROUND_UP_2((int)rect_params.top);
gint src_width = GST_ROUND_DOWN_2((int)rect_params.width);
gint src_height = GST_ROUND_DOWN_2((int)rect_params.height);
// Maintain aspect ratio
double hdest = processing_width * src_height / (double)src_width;
double wdest = processing_height * src_width / (double)src_height;
guint dest_width, dest_height;
if (hdest <= processing_height) {
dest_width = processing_width;
dest_height = hdest;
} else {
dest_width = wdest;
dest_height = processing_height;
}
// Configure transform session parameters for the transformation
NvBufSurfTransformConfigParams transform_config_params;
transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
transform_config_params.gpu_id = gpu_id_;
transform_config_params.cuda_stream = cuda_stream_;
// Set the transform session parameters for the conversions executed in this thread.
if (auto err = NvBufSurfTransformSetSessionParams(&transform_config_params);
err != NvBufSurfTransformError_Success) {
spdlog::error("NvBufSurfTransformSetSessionParams failed with error {}", err);
return tl::unexpected {GST_FLOW_ERROR};
}
// Calculate scaling ratio while maintaining aspect ratio
ratio = MIN(1.0 * dest_width / src_width, 1.0 * dest_height / src_height);
if ((rect_params.width == 0) || (rect_params.height == 0)) {
spdlog::error("[get_opencv_mat]:crop_rect_params dimensions are zero");
return tl::unexpected {GST_FLOW_ERROR};
}
#ifdef __aarch64__
if (ratio <= 1.0 / 16 || ratio >= 16.0) {
// Currently cannot scale by ratio > 16 or < 1/16 for Jetson
spdlog::error("[get_opencv_mat] ratio {} not in range of [.0625 : 16]", ratio);
return tl::unexpected {GST_FLOW_ERROR};
}
#endif
// Set the transform ROIs for source and destination
NvBufSurfTransformRect src_rect = {(guint)src_top, (guint)src_left, (guint)src_width,
(guint)src_height};
NvBufSurfTransformRect dst_rect = {0, 0, (guint)dest_width, (guint)dest_height};
// Set the transform parameters
NvBufSurfTransformParams transform_params;
transform_params.src_rect = &src_rect;
transform_params.dst_rect = &dst_rect;
transform_params.transform_flag =
NVBUFSURF_TRANSFORM_FILTER | NVBUFSURF_TRANSFORM_CROP_SRC | NVBUFSURF_TRANSFORM_CROP_DST;
transform_params.transform_filter = NvBufSurfTransformInter_Default;
// Memset the memory
NvBufSurfaceMemSet(inter_buf_, idx, 0, 0);
// spdlog::debug("Scaling and converting input buffer");
// Transformation scaling+format conversion if any. Not sure if the transform is actually doing
// anything other than a copy? Maybe converting from NV12 to RGBA? Can we convert from NV12 to
// BGR directly so only one conversion is needed?
if (auto err = NvBufSurfTransform(&ip_surf, inter_buf_, &transform_params);
err != NvBufSurfTransformError_Success) {
spdlog::error("NvBufSurfTransform failed with error {} while converting buffer", err);
return tl::unexpected {GST_FLOW_ERROR};
}
// Map the buffer so that it can be accessed by CPU
if (NvBufSurfaceMap(inter_buf_, idx, 0, NVBUF_MAP_READ) != 0) {
return tl::unexpected {GST_FLOW_ERROR};
}
// Cache the mapped data for CPU access
NvBufSurfaceSyncForCpu(inter_buf_, idx, 0);
// Use openCV to remove padding and convert RGBA to BGR. Can be skipped if
// algorithm can handle padded RGBA data.
const auto in_mat =
cv::Mat(processing_height, processing_width, CV_8UC4,
inter_buf_->surfaceList[0].mappedAddr.addr[0], inter_buf_->surfaceList[0].pitch);
cv::Mat image_bgr;
cv::cvtColor(in_mat, image_bgr, cv::COLOR_RGBA2BGR);
if (NvBufSurfaceUnMap(inter_buf_, idx, 0)) {
return tl::unexpected {GST_FLOW_ERROR};
}
return image_bgr;
}
which is called in the src pad probe callback from nvinfer:
static GstPadProbeReturn pgie_src_pad_buffer_probe(GstPad* pad, GstPadProbeInfo* info,
gpointer u_data) {
GstCaps* caps = gst_pad_get_current_caps(pad);
GstVideoInfo video_info = {};
if (!gst_video_info_from_caps(&video_info, caps)) {
logger_->error("[ProcessFrame] failed to get video_info");
return (GstPadProbeReturn)GST_FLOW_ERROR;
}
gst_caps_unref(caps);
// NvDsDisplayMeta *display_meta = NULL;
GstBuffer* buf = (GstBuffer*)info->data;
GstMapInfo inmap = GST_MAP_INFO_INIT;
if (!gst_buffer_map(buf, &inmap, GST_MAP_READ)) {
GST_ERROR("input buffer mapinfo failed");
return (GstPadProbeReturn)GST_FLOW_ERROR;
}
if (inter_buf_ == nullptr) {
logger_->info("[ProcessFrame] initializing Frame buffer");
NvBufSurfaceCreateParams create_params;
create_params.gpuId = gpu_id_;
create_params.width = video_info.width; // dsexample->processing_width;
create_params.height = video_info.height; // dsexample->processing_height;
create_params.size = 0;
create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
create_params.layout = NVBUF_LAYOUT_PITCH;
#ifdef __aarch64__
create_params.memType = NVBUF_MEM_DEFAULT;
#else
create_params.memType = NVBUF_MEM_CUDA_UNIFIED;
#endif
if (NvBufSurfaceCreate(&inter_buf_, 1, &create_params) != 0) {
logger_->error("Error: Could not allocate internal buffer for dsexample");
GST_ERROR("Error: Could not allocate internal buffer for dsexample");
return (GstPadProbeReturn)GST_FLOW_ERROR;
}
}
NvBufSurface* ip_surf = (NvBufSurface*)inmap.data;
gst_buffer_unmap(buf, &inmap);
const int index = 0; // at the moment our batch size == 1 so to simplify things, start there
// instead of looping through buffers, etc.
double scale_ratio = 1.0;
const auto frame_width = ip_surf->surfaceList[0].width;
const auto frame_height = ip_surf->surfaceList[0].height;
cv::Mat frame;
if (const auto ret =
vp->get_opencv_mat(*ip_surf, index, scale_ratio, frame_width, frame_height);
!ret.has_value()) {
return (GstPadProbeReturn)ret.error();
} else {
frame = ret.value();
}
If you have any suggestions for simplifying this (e.g. convert directly from nv12 to bgr) that would be great.
