• Hardware Platform (Jetson / GPU)
Jetson
• DeepStream Version
6.1.1
• JetPack Version (valid for Jetson only)
5.2
• TensorRT Version
8.4.1.5
• Issue Type( questions, new requirements, bugs)
questions
• Requirement details( This is for new requirement. Including the module name-for which plugin or for which sample application, the function description)
i’m writing a custom gst-infer plugin that do the rectification of the faces read by the primary retinaface using landmarks so i’m modifying the convert_batch_and_push_to_input_thread function
i’m able to get the cropped faces but i also need the whole frame to do the rectification
static gboolean
convert_batch_and_push_to_input_thread_face(
GstNvInfer *nvinfer,
GstNvInferBatch *batch,
GstNvInferMemory *mem,
NvDsFrameMeta *frame_meta,
NvDsObjectMeta *object_meta,
NvOSD_RectParams *crop_rect_params,
float face_landmarks[][2]) {
NvBufSurfTransform_Error err = NvBufSurfTransformError_Success;
std::string nvtx_str;
/* Set the transform session parameters for the conversions executed in this
* thread. */
err = NvBufSurfTransformSetSessionParams(&nvinfer->transform_config_params);
if (err != NvBufSurfTransformError_Success) {
GST_ELEMENT_ERROR(nvinfer, STREAM, FAILED,
("NvBufSurfTransformSetSessionParams failed with error %d", err), (NULL));
return FALSE;
}
nvtxEventAttributes_t eventAttrib = {0};
eventAttrib.version = NVTX_VERSION;
eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
eventAttrib.colorType = NVTX_COLOR_ARGB;
eventAttrib.color = 0xFFFF0000;
eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
nvtx_str = "convert_buf batch_num=" + std::to_string(nvinfer->current_batch_num);
eventAttrib.message.ascii = nvtx_str.c_str();
nvtxDomainRangePushEx(nvinfer->nvtx_domain, &eventAttrib);
if (batch->frames.size() > 0) {
err = NvBufSurfTransform(&nvinfer->tmp_surf, mem->surf, &nvinfer->transform_params);
}
nvtxDomainRangePop(nvinfer->nvtx_domain);
if (err != NvBufSurfTransformError_Success) {
GST_ELEMENT_ERROR(nvinfer, STREAM, FAILED,
("NvBufSurfTransform failed with error %d while converting buffer", err),
(NULL));
return FALSE;
}
//TODO: Saves images only when we are not processing a full frame -> secondary inference
if ((!nvinfer->process_full_frame) && (mem->surf->numFilled > 0) || true) {
// First convert the surface buffer to 8-bit gray
// Transformation parameters
NvBufSurfTransformParams surf_transform_params = {
.transform_flag = NVBUFSURF_TRANSFORM_FILTER,
.transform_flip = NvBufSurfTransform_None,
.transform_filter = NvBufSurfTransformInter_Default,
.src_rect = nullptr,
.dst_rect = nullptr
};
// Surface creation parameters width and height is 112,112
NvBufSurfaceCreateParams nvbufsurface_create_params{
.gpuId = mem->surf->gpuId,
.width = (gint) mem->surf->surfaceList[0].width,
.height = (gint) mem->surf->surfaceList[0].height,
.size = 0,
.isContiguous = true,
//.colorFormat = NVBUF_COLOR_FORMAT_GRAY8,
.colorFormat = NVBUF_COLOR_FORMAT_BGRA,
//.colorFormat = NVBUF_COLOR_FORMAT_BGRA,//works in Jetson
//.colorFormat = NVBUF_COLOR_FORMAT_BGR, //does not work in Jetson
.layout = NVBUF_LAYOUT_PITCH,
.memType = NVBUF_MEM_DEFAULT
};
NvBufSurface *surf_bgra = NULL;
if (NvBufSurfaceCreate(&surf_bgra, mem->surf->batchSize, &nvbufsurface_create_params) != 0) {
std::cerr << "Failed to allocate space for surface 'surf_bgra'" << std::endl;
}
surf_bgra->numFilled = mem->surf->numFilled;
int err = 0;
if ((err = NvBufSurfTransform(mem->surf, surf_bgra, &surf_transform_params)) !=
NvBufSurfTransformError_Success) {
std::cerr << "Failed to transform 'mem->surf' into bgra, error code: " << err << std::endl;
} else {
static unsigned int secondary_object_counter = 0;
cv::Mat cropped;
#if defined(__arm__) || defined (__aarch64__)
cropped = nvdsutils::get_surf_bgr_from_bgra(surf_bgra, false);
#else
cropped = nvdsutils::get_surf_bgr_from_bgra(surf_bgra, true);
#endif
auto start = std::chrono::system_clock::now();
char img_name[500] = "";
std::cout << "writing image for object_id : " << object_meta->object_id << "\n";
sprintf(img_name, "videos/images/origin-%"G_GUINT64_FORMAT".png", object_meta->object_id);
printf("img_name : %s\n", img_name);
cv::imwrite(img_name, cropped);
}
NvBufSurfaceDestroy(surf_bgra);
}
LockGMutex locker(nvinfer->process_lock);
/* Push the batch info structure in the processing queue and notify the output
* thread that a new batch has been queued. */
g_queue_push_tail(nvinfer->input_queue, batch);
g_cond_broadcast(&nvinfer->process_cond);
return TRUE;
}
cv::Mat get_surf_bgr_from_bgra(NvBufSurface* surf, bool use_pitch_alignment)
{
cv::Mat out_mat;
NvBufSurface* drawMe = nullptr;
if((surf == nullptr) || (surf->numFilled <=0))
{
std::cerr << "get_surf_bgr_from_bgra: surf is nullptr or numFilled is 0" << std::endl;
return out_mat;
}
if(surf->surfaceList[0].colorFormat != NVBUF_COLOR_FORMAT_BGRA)
{
std::cerr << "get_surf_bgr_from_bgra: only NVBUF_COLOR_FORMAT_BGRA is supported, current format is " << surf->surfaceList[0].colorFormat << std::endl;
std::cerr << NVBUF_COLOR_FORMAT_BGRA << "\n";
return out_mat;
}
// Allocate system memory and copy the surface buffers to the allocated memory
NvBufSurfaceCreateParams surf_create_params = {
.gpuId = surf->gpuId,
.width = surf->surfaceList[0].width,
.height = surf->surfaceList[0].height,
.size = 0,
.isContiguous = true,
.colorFormat = NVBUF_COLOR_FORMAT_BGRA,
.layout = NVBUF_LAYOUT_PITCH,
.memType = NVBUF_MEM_SYSTEM
};
// Allocate memory
if(NvBufSurfaceCreate(&drawMe, surf->batchSize, &surf_create_params) != 0)
{
std::cerr << "get_surf_bgr_from_bgra: failed to allocate memory" << std::endl;
return out_mat;
}
// Copy the buffers to the allocated memory
if(NvBufSurfaceCopy(surf, drawMe) != 0)
{
std::cerr << "get_surf_bgr_from_bgra: failed to copy buffers" << std::endl;
NvBufSurfaceDestroy(drawMe);
return out_mat;
}
for(std::size_t i = 0; i < surf->numFilled; i++)
{
cv::Mat mapped;
if(use_pitch_alignment)
{
mapped = cv::Mat(drawMe->surfaceList[i].height, drawMe->surfaceList[i].width, CV_8UC4, drawMe->surfaceList[i].dataPtr, drawMe->surfaceList[i].pitch);
}else{
mapped = cv::Mat(drawMe->surfaceList[i].height, drawMe->surfaceList[i].width, CV_8UC4, drawMe->surfaceList[i].dataPtr);
}
try{
out_mat = cv::Mat(cv::Size(drawMe->surfaceList[i].width, drawMe->surfaceList[i].height), CV_8UC3);
cv::cvtColor(mapped, out_mat, cv::COLOR_BGRA2BGR);
} catch(const std::exception& e)
{
NvBufSurfaceDestroy(drawMe);
std::cerr << "write_surf_rgb_to_disk exception thrown during saving the image: " << e.what() << std::endl;
return out_mat;
}
}
// Release memory
NvBufSurfaceDestroy(drawMe);
return out_mat;
}
i can correctly get the cropped face and save it to a file, i was wondering, how i get also the full frame i need to do the rectification?
thank you, William