Here is the full file as well as the header
gstdsdetectcolors.cpp
type or paste code h/**
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include <string.h>
#include <string>
#include <sstream>
#include <iostream>
#include <ostream>
#include <fstream>
#include "gstdsdetectcolors.h"
#include <sys/time.h>
#include "nvdsmeta_schema.h"
#include "nvdsmeta.h"
#include "gstnvdsmeta.h"
#include "nvdspreprocess_meta.h"
#include "nvds_roi_meta.h"
GST_DEBUG_CATEGORY_STATIC (gst_dsdetectcolors_debug);
#define GST_CAT_DEFAULT gst_dsdetectcolors_debug
#define USE_EGLIMAGE 1
/* enable to write transformed cvmat to files */
/* #define DSDETECTCOLORS_DEBUG */
static GQuark _dsmeta_quark = 0;
static GMutex lock;
//using json = nlohmann::json;
/* Enum to identify properties */
enum
{
PROP_0,
PROP_UNIQUE_ID,
PROP_PROCESSING_WIDTH,
PROP_PROCESSING_HEIGHT,
PROP_PROCESS_FULL_FRAME,
PROP_BLUR_OBJECTS,
PROP_GPU_DEVICE_ID
};
#define CHECK_NVDS_MEMORY_AND_GPUID(object, surface) \
({ int _errtype=0;\
do { \
if ((surface->memType == NVBUF_MEM_DEFAULT || surface->memType == NVBUF_MEM_CUDA_DEVICE) && \
(surface->gpuId != object->gpu_id)) { \
GST_ELEMENT_ERROR (object, RESOURCE, FAILED, \
("Input surface gpu-id doesnt match with configured gpu-id for element," \
" please allocate input using unified memory, or use same gpu-ids"),\
("surface-gpu-id=%d,%s-gpu-id=%d",surface->gpuId,GST_ELEMENT_NAME(object),\
object->gpu_id)); \
_errtype = 1;\
} \
} while(0); \
_errtype; \
})
/* Default values for properties */
#define DEFAULT_UNIQUE_ID 15
#define DEFAULT_PROCESSING_WIDTH 480
#define DEFAULT_PROCESSING_HEIGHT 480
#define DEFAULT_PROCESS_FULL_FRAME FALSE
#define DEFAULT_BLUR_OBJECTS FALSE
#define DEFAULT_GPU_ID 0
#define RGB_BYTES_PER_PIXEL 3
#define RGBA_BYTES_PER_PIXEL 4
#define Y_BYTES_PER_PIXEL 1
#define UV_BYTES_PER_PIXEL 2
#define MIN_INPUT_OBJECT_WIDTH 16
#define MIN_INPUT_OBJECT_HEIGHT 16
#define MAX_TIME_STAMP_LEN 32
#define PGIE_CLASS_ID_MARKER 1
#define INPUT_SIZE 36 //1 * 9 * 4;
#define CHECK_NPP_STATUS(npp_status,error_str) do { \
if ((npp_status) != NPP_SUCCESS) { \
g_print ("Error: %s in %s at line %d: NPP Error %d\n", \
error_str, __FILE__, __LINE__, npp_status); \
goto error; \
} \
} while (0)
#define CHECK_CUDA_STATUS(cuda_status,error_str) do { \
if ((cuda_status) != cudaSuccess) { \
g_print ("Error: %s in %s at line %d (%s)\n", \
error_str, __FILE__, __LINE__, cudaGetErrorName(cuda_status)); \
goto error; \
} \
} while (0)
/* By default NVIDIA Hardware allocated memory flows through the pipeline. We
* will be processing on this type of memory only. */
#define GST_CAPS_FEATURE_MEMORY_NVMM "memory:NVMM"
static GstStaticPadTemplate gst_dsdetectcolors_sink_template =
GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_NVMM,
"{ NV12, RGBA, I420 }")));
static GstStaticPadTemplate gst_dsdetectcolors_src_template =
GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_NVMM,
"{ NV12, RGBA, I420 }")));
/* Define our element type. Standard GObject/GStreamer boilerplate stuff */
#define gst_dsdetectcolors_parent_class parent_class
G_DEFINE_TYPE (GstDsDetectColors, gst_dsdetectcolors, GST_TYPE_BASE_TRANSFORM);
static void gst_dsdetectcolors_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec);
static void gst_dsdetectcolors_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec);
static gboolean gst_dsdetectcolors_set_caps (GstBaseTransform * btrans,
GstCaps * incaps, GstCaps * outcaps);
static gboolean gst_dsdetectcolors_start (GstBaseTransform * btrans);
static gboolean gst_dsdetectcolors_stop (GstBaseTransform * btrans);
static GstFlowReturn gst_dsdetectcolors_transform_ip (GstBaseTransform *
btrans, GstBuffer * inbuf);
static void
attach_metadata_full_frame (GstDsDetectColors * dsdetectcolors, NvDsFrameMeta *frame_meta,
gdouble scale_ratio, DsDetectColorsOutput * output, guint batch_id);
static void attach_metadata_object (GstDsDetectColors * dsdetectcolors,
NvDsObjectMeta * obj_meta, DsDetectColorsOutput * output);
/* Install properties, set sink and src pad capabilities, override the required
* functions of the base class, These are common to all instances of the
* element.
*/
static void
gst_dsdetectcolors_class_init (GstDsDetectColorsClass * klass)
{
GObjectClass *gobject_class;
GstElementClass *gstelement_class;
GstBaseTransformClass *gstbasetransform_class;
/* Indicates we want to use DS buf api */
g_setenv ("DS_NEW_BUFAPI", "1", TRUE);
gobject_class = (GObjectClass *) klass;
gstelement_class = (GstElementClass *) klass;
gstbasetransform_class = (GstBaseTransformClass *) klass;
/* Overide base class functions */
gobject_class->set_property = GST_DEBUG_FUNCPTR (gst_dsdetectcolors_set_property);
gobject_class->get_property = GST_DEBUG_FUNCPTR (gst_dsdetectcolors_get_property);
gstbasetransform_class->set_caps = GST_DEBUG_FUNCPTR (gst_dsdetectcolors_set_caps);
gstbasetransform_class->start = GST_DEBUG_FUNCPTR (gst_dsdetectcolors_start);
gstbasetransform_class->stop = GST_DEBUG_FUNCPTR (gst_dsdetectcolors_stop);
gstbasetransform_class->transform_ip =
GST_DEBUG_FUNCPTR (gst_dsdetectcolors_transform_ip);
/* Install properties */
g_object_class_install_property (gobject_class, PROP_UNIQUE_ID,
g_param_spec_uint ("unique-id",
"Unique ID",
"Unique ID for the element. Can be used to identify output of the"
" element", 0, G_MAXUINT, DEFAULT_UNIQUE_ID, (GParamFlags)
(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_PROCESSING_WIDTH,
g_param_spec_int ("processing-width",
"Processing Width",
"Width of the input buffer to algorithm",
1, G_MAXINT, DEFAULT_PROCESSING_WIDTH, (GParamFlags)
(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_PROCESSING_HEIGHT,
g_param_spec_int ("processing-height",
"Processing Height",
"Height of the input buffer to algorithm",
1, G_MAXINT, DEFAULT_PROCESSING_HEIGHT, (GParamFlags)
(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_PROCESS_FULL_FRAME,
g_param_spec_boolean ("full-frame",
"Full frame",
"Enable to process full frame or disable to process objects detected"
"by primary detector", DEFAULT_PROCESS_FULL_FRAME, (GParamFlags)
(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_BLUR_OBJECTS,
g_param_spec_boolean ("blur-objects",
"Blur Objects",
"Enable to blur the objects detected in full-frame=0 mode"
"by primary detector", DEFAULT_BLUR_OBJECTS, (GParamFlags)
(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_GPU_DEVICE_ID,
g_param_spec_uint ("gpu-id",
"Set GPU Device ID",
"Set GPU Device ID", 0,
G_MAXUINT, 0,
GParamFlags
(G_PARAM_READWRITE |
G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY)));
/* Set sink and src pad capabilities */
gst_element_class_add_pad_template (gstelement_class,
gst_static_pad_template_get (&gst_dsdetectcolors_src_template));
gst_element_class_add_pad_template (gstelement_class,
gst_static_pad_template_get (&gst_dsdetectcolors_sink_template));
/* Set metadata describing the element */
gst_element_class_set_details_simple (gstelement_class,
"DsDetectColors plugin",
"DsDetectColors Plugin",
"Process a 3rdparty example algorithm on objects / full frame",
"NVIDIA Corporation. Post on Deepstream for Tesla forum for any queries "
"@ https://devtalk.nvidia.com/default/board/209/");
}
static void
gst_dsdetectcolors_init (GstDsDetectColors * dsdetectcolors)
{
GstBaseTransform *btrans = GST_BASE_TRANSFORM (dsdetectcolors);
/* We will not be generating a new buffer. Just adding / updating
* metadata. */
gst_base_transform_set_in_place (GST_BASE_TRANSFORM (btrans), TRUE);
/* We do not want to change the input caps. Set to passthrough. transform_ip
* is still called. */
gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (btrans), TRUE);
/* Initialize all property variables to default values */
dsdetectcolors->unique_id = DEFAULT_UNIQUE_ID;
dsdetectcolors->processing_width = DEFAULT_PROCESSING_WIDTH;
dsdetectcolors->processing_height = DEFAULT_PROCESSING_HEIGHT;
dsdetectcolors->process_full_frame = DEFAULT_PROCESS_FULL_FRAME;
dsdetectcolors->blur_objects = DEFAULT_BLUR_OBJECTS;
dsdetectcolors->gpu_id = DEFAULT_GPU_ID;
/* This quark is required to identify NvDsMeta when iterating through
* the buffer metadatas */
if (!_dsmeta_quark)
_dsmeta_quark = g_quark_from_static_string (NVDS_META_STRING);
}
/* Function called when a property of the element is set. Standard boilerplate.
*/
static void
gst_dsdetectcolors_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstDsDetectColors *dsdetectcolors = GST_DSDETECTCOLORS (object);
switch (prop_id) {
case PROP_UNIQUE_ID:
dsdetectcolors->unique_id = g_value_get_uint (value);
break;
case PROP_PROCESSING_WIDTH:
dsdetectcolors->processing_width = g_value_get_int (value);
break;
case PROP_PROCESSING_HEIGHT:
dsdetectcolors->processing_height = g_value_get_int (value);
break;
case PROP_PROCESS_FULL_FRAME:
dsdetectcolors->process_full_frame = g_value_get_boolean (value);
break;
case PROP_BLUR_OBJECTS:
dsdetectcolors->blur_objects = g_value_get_boolean (value);
break;
case PROP_GPU_DEVICE_ID:
dsdetectcolors->gpu_id = g_value_get_uint (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
/* Function called when a property of the element is requested. Standard
* boilerplate.
*/
static void
gst_dsdetectcolors_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstDsDetectColors *dsdetectcolors = GST_DSDETECTCOLORS (object);
switch (prop_id) {
case PROP_UNIQUE_ID:
g_value_set_uint (value, dsdetectcolors->unique_id);
break;
case PROP_PROCESSING_WIDTH:
g_value_set_int (value, dsdetectcolors->processing_width);
break;
case PROP_PROCESSING_HEIGHT:
g_value_set_int (value, dsdetectcolors->processing_height);
break;
case PROP_PROCESS_FULL_FRAME:
g_value_set_boolean (value, dsdetectcolors->process_full_frame);
break;
case PROP_BLUR_OBJECTS:
g_value_set_boolean (value, dsdetectcolors->blur_objects);
break;
case PROP_GPU_DEVICE_ID:
g_value_set_uint (value, dsdetectcolors->gpu_id);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
/**
* Initialize all resources and start the output thread
*/
static gboolean
gst_dsdetectcolors_start (GstBaseTransform * btrans)
{
GstDsDetectColors *dsdetectcolors = GST_DSDETECTCOLORS (btrans);
NvBufSurfaceCreateParams create_params;
DsDetectColorsInitParams init_params =
{ dsdetectcolors->processing_width, dsdetectcolors->processing_height,
dsdetectcolors->process_full_frame
};
GstQuery *queryparams = NULL;
guint batch_size = 1;
int val = -1;
/* Algorithm specific initializations and resource allocation. */
dsdetectcolors->dsdetectcolorslib_ctx = DsDetectColorsCtxInit (&init_params);
GST_DEBUG_OBJECT (dsdetectcolors, "ctx lib %p \n", dsdetectcolors->dsdetectcolorslib_ctx);
CHECK_CUDA_STATUS (cudaSetDevice (dsdetectcolors->gpu_id),
"Unable to set cuda device");
cudaDeviceGetAttribute (&val, cudaDevAttrIntegrated, dsdetectcolors->gpu_id);
dsdetectcolors->is_integrated = val;
dsdetectcolors->batch_size = 1;
queryparams = gst_nvquery_batch_size_new ();
if (gst_pad_peer_query (GST_BASE_TRANSFORM_SINK_PAD (btrans), queryparams)
|| gst_pad_peer_query (GST_BASE_TRANSFORM_SRC_PAD (btrans), queryparams)) {
if (gst_nvquery_batch_size_parse (queryparams, &batch_size)) {
dsdetectcolors->batch_size = batch_size;
}
}
GST_DEBUG_OBJECT (dsdetectcolors, "Setting batch-size %d \n",
dsdetectcolors->batch_size);
gst_query_unref (queryparams);
if (dsdetectcolors->process_full_frame && dsdetectcolors->blur_objects) {
GST_ERROR ("Error: does not support blurring while processing full frame");
goto error;
}
CHECK_CUDA_STATUS (cudaStreamCreate (&dsdetectcolors->cuda_stream),
"Could not create cuda stream");
if (dsdetectcolors->inter_buf)
NvBufSurfaceDestroy (dsdetectcolors->inter_buf);
dsdetectcolors->inter_buf = NULL;
/* An intermediate buffer for NV12/RGBA to BGR conversion will be
* required. Can be skipped if custom algorithm can work directly on NV12/RGBA. */
create_params.gpuId = dsdetectcolors->gpu_id;
create_params.width = dsdetectcolors->processing_width;
create_params.height = dsdetectcolors->processing_height;
create_params.size = 0;
create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
create_params.layout = NVBUF_LAYOUT_PITCH;
if(dsdetectcolors->is_integrated) {
create_params.memType = NVBUF_MEM_DEFAULT;
}
else {
create_params.memType = NVBUF_MEM_CUDA_PINNED;
}
if (NvBufSurfaceCreate (&dsdetectcolors->inter_buf, 1,
&create_params) != 0) {
GST_ERROR ("Error: Could not allocate internal buffer for dsdetectcolors");
goto error;
}
/* Create host memory for storing converted/scaled interleaved RGB data */
CHECK_CUDA_STATUS (cudaMallocHost (&dsdetectcolors->host_rgb_buf,
dsdetectcolors->processing_width * dsdetectcolors->processing_height *
RGB_BYTES_PER_PIXEL), "Could not allocate cuda host buffer");
GST_DEBUG_OBJECT (dsdetectcolors, "allocated cuda buffer %p \n",
dsdetectcolors->host_rgb_buf);
/* CV Mat containing interleaved RGB data. This call does not allocate memory.
* It uses host_rgb_buf as data. */
dsdetectcolors->cvmat =
new cv::cuda::GpuMat (dsdetectcolors->processing_height, dsdetectcolors->processing_width,
CV_8UC3, dsdetectcolors->host_rgb_buf,
dsdetectcolors->processing_width * RGB_BYTES_PER_PIXEL);
if (!dsdetectcolors->cvmat)
goto error;
GST_DEBUG_OBJECT (dsdetectcolors, "created CV Mat\n");
return TRUE;
error:
if (dsdetectcolors->host_rgb_buf) {
cudaFreeHost (dsdetectcolors->host_rgb_buf);
dsdetectcolors->host_rgb_buf = NULL;
}
if (dsdetectcolors->cuda_stream) {
cudaStreamDestroy (dsdetectcolors->cuda_stream);
dsdetectcolors->cuda_stream = NULL;
}
if (dsdetectcolors->dsdetectcolorslib_ctx)
DsDetectColorsCtxDeinit (dsdetectcolors->dsdetectcolorslib_ctx);
return FALSE;
}
/**
* Stop the output thread and free up all the resources
*/
static gboolean
gst_dsdetectcolors_stop (GstBaseTransform * btrans)
{
GstDsDetectColors *dsdetectcolors = GST_DSDETECTCOLORS (btrans);
if (dsdetectcolors->inter_buf)
NvBufSurfaceDestroy(dsdetectcolors->inter_buf);
dsdetectcolors->inter_buf = NULL;
if (dsdetectcolors->cuda_stream)
cudaStreamDestroy (dsdetectcolors->cuda_stream);
dsdetectcolors->cuda_stream = NULL;
delete dsdetectcolors->cvmat;
dsdetectcolors->cvmat = NULL;
if (dsdetectcolors->host_rgb_buf) {
cudaFreeHost (dsdetectcolors->host_rgb_buf);
dsdetectcolors->host_rgb_buf = NULL;
}
GST_DEBUG_OBJECT (dsdetectcolors, "deleted CV Mat \n");
/* Deinit the algorithm library */
DsDetectColorsCtxDeinit (dsdetectcolors->dsdetectcolorslib_ctx);
dsdetectcolors->dsdetectcolorslib_ctx = NULL;
GST_DEBUG_OBJECT (dsdetectcolors, "ctx lib released \n");
return TRUE;
}
/**
* Called when source / sink pad capabilities have been negotiated.
*/
static gboolean
gst_dsdetectcolors_set_caps (GstBaseTransform * btrans, GstCaps * incaps,
GstCaps * outcaps)
{
GstDsDetectColors *dsdetectcolors = GST_DSDETECTCOLORS (btrans);
/* Save the input video information, since this will be required later. */
gst_video_info_from_caps (&dsdetectcolors->video_info, incaps);
if (dsdetectcolors->blur_objects && !dsdetectcolors->process_full_frame) {
/* requires RGBA format for blurring the objects in opencv */
if (dsdetectcolors->video_info.finfo->format != GST_VIDEO_FORMAT_RGBA) {
GST_ELEMENT_ERROR (dsdetectcolors, STREAM, FAILED,
("input format should be RGBA when using blur-objects property"), (NULL));
goto error;
}
}
return TRUE;
error:
return FALSE;
}
/**
* Scale the entire frame to the processing resolution maintaining aspect ratio.
* Or crop and scale objects to the processing resolution maintaining the aspect
* ratio. Remove the padding required by hardware and convert from RGBA to RGB
* using openCV. These steps can be skipped if the algorithm can work with
* padded data and/or can work with RGBA.
*/
static GstFlowReturn
get_converted_mat (GstDsDetectColors * dsdetectcolors, NvBufSurface *input_buf, gint idx,
NvOSD_RectParams * crop_rect_params, gdouble & ratio, gint input_width,
gint input_height, cv::cuda::GpuMat &crop)
{
NvBufSurfTransform_Error err;
NvBufSurfTransformConfigParams transform_config_params;
NvBufSurfTransformParams transform_params;
NvBufSurfTransformRect src_rect;
NvBufSurfTransformRect dst_rect;
NvBufSurface ip_surf;
//cv::Mat in_mat, out_mat;
cv::cuda::GpuMat gpuMat;
ip_surf = *input_buf;
ip_surf.numFilled = ip_surf.batchSize = 1;
ip_surf.surfaceList = &(input_buf->surfaceList[idx]);
gint src_left = GST_ROUND_UP_2((unsigned int)crop_rect_params->left);
gint src_top = GST_ROUND_UP_2((unsigned int)crop_rect_params->top);
gint src_width = GST_ROUND_DOWN_2((unsigned int)crop_rect_params->width);
gint src_height = GST_ROUND_DOWN_2((unsigned int)crop_rect_params->height);
//g_print("ltwh = %d %d %d %d \n", src_left, src_top, src_width, src_height);
guint dest_width, dest_height;
dest_width = src_width;
dest_height = src_height;
NvBufSurface *nvbuf;
NvBufSurfaceCreateParams create_params;
create_params.gpuId = dsdetectcolors->gpu_id;
create_params.width = dest_width;
create_params.height = dest_height;
create_params.size = 0;
create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
create_params.layout = NVBUF_LAYOUT_PITCH;
#ifdef __aarch64__
create_params.memType = NVBUF_MEM_DEFAULT;
#else
create_params.memType = NVBUF_MEM_CUDA_UNIFIED;
#endif
NvBufSurfaceCreate (&nvbuf, 1, &create_params);
// Configure transform session parameters for the transformation
transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
transform_config_params.gpu_id = dsdetectcolors->gpu_id;
transform_config_params.cuda_stream = dsdetectcolors->cuda_stream;
// Set the transform session parameters for the conversions executed in this
// thread.
err = NvBufSurfTransformSetSessionParams (&transform_config_params);
if (err != NvBufSurfTransformError_Success) {
GST_ELEMENT_ERROR (dsdetectcolors, STREAM, FAILED,
("NvBufSurfTransformSetSessionParams failed with error %d", err), (NULL));
goto error;
}
// Calculate scaling ratio while maintaining aspect ratio
ratio = MIN (1.0 * dest_width/ src_width, 1.0 * dest_height / src_height);
if ((crop_rect_params->width == 0) || (crop_rect_params->height == 0)) {
GST_ELEMENT_ERROR (dsdetectcolors, STREAM, FAILED,
("%s:crop_rect_params dimensions are zero",__func__), (NULL));
goto error;
}
#ifdef __aarch64__
if (ratio <= 1.0 / 16 || ratio >= 16.0) {
// Currently cannot scale by ratio > 16 or < 1/16 for Jetson
goto error;
}
#endif
// Set the transform ROIs for source and destination
src_rect = {(guint)src_top, (guint)src_left, (guint)src_width, (guint)src_height};
dst_rect = {0, 0, (guint)dest_width, (guint)dest_height};
// Set the transform parameters
transform_params.src_rect = &src_rect;
transform_params.dst_rect = &dst_rect;
transform_params.transform_flag =
NVBUFSURF_TRANSFORM_FILTER | NVBUFSURF_TRANSFORM_CROP_SRC |
NVBUFSURF_TRANSFORM_CROP_DST;
transform_params.transform_filter = NvBufSurfTransformInter_Default;
#ifdef __aarch64__
// To use the converted buffer in CUDA, create an EGLImage and then use
// CUDA-EGL interop APIs
if (USE_EGLIMAGE) {
if (NvBufSurfaceMapEglImage (dsdetectcolors->inter_buf, 0) !=0 ) {
goto error;
}
//static bool create_filter = true;
//static cv::Ptr< cv::cuda::Filter > filter;
CUresult status;
CUeglFrame eglFrame;
CUgraphicsResource pResource = NULL;
cudaFree(0);
status = cuGraphicsEGLRegisterImage(&pResource,
dsdetectcolors->inter_buf->surfaceList[0].mappedAddr.eglImage,
CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE);
if (status != CUDA_SUCCESS) {
GST_ERROR("cuGraphicsEGLRegisterImage failed: %d", status);
goto error;
}
status = cuGraphicsResourceGetMappedEglFrame(&eglFrame, pResource, 0, 0);
if (status != CUDA_SUCCESS) {
GST_ERROR("cuGraphicsResourceGetMappedEglFrame failed: %d", status);
cuGraphicsUnregisterResource(pResource);
goto error;
}
status = cuCtxSynchronize();
if (status != CUDA_SUCCESS) {
GST_ERROR("cuCtxSynchronize failed: %d", status);
cuGraphicsUnregisterResource(pResource);
goto error;
}
// if (create_filter) {
// filter = cv::cuda::createSobelFilter(CV_8UC4, CV_8UC4, 1, 0, 3, 1, cv::BORDER_DEFAULT);
// //filter = cv::cuda::createGaussianFilter(CV_8UC4, CV_8UC4, cv::Size(31,31), 0, 0, cv::BORDER_DEFAULT);
// create_filter = false;
// }
cv::cuda::GpuMat d_mat(dest_height, dest_width, CV_8UC4, eglFrame.frame.pPitch[0]);
//filter->apply (d_mat, d_mat);
status = cuCtxSynchronize();
if (status != CUDA_SUCCESS) {
GST_ERROR("cuCtxSynchronize failed: %d", status);
cuGraphicsUnregisterResource(pResource);
goto error;
}
cv::cuda::cvtColor (d_mat, crop, cv::COLOR_RGBA2BGR);
status = cuGraphicsUnregisterResource(pResource);
if (status != CUDA_SUCCESS) {
GST_ERROR("cuGraphicsUnregisterResource failed: %d", status);
}
// apply back to the original buffer
transform_params.src_rect = &dst_rect;
transform_params.dst_rect = &src_rect;
NvBufSurfTransform (dsdetectcolors->inter_buf, &ip_surf, &transform_params);
// Destroy the EGLImage
NvBufSurfaceUnMapEglImage (dsdetectcolors->inter_buf, 0);
}
#endif
/* We will first convert only the Region of Interest (the entire frame or the
* object bounding box) to RGB and then scale the converted RGB frame to
* processing resolution. */
return GST_FLOW_OK;
error:
return GST_FLOW_ERROR;
}
/**
* Called when element recieves an input buffer from upstream element.
*/
static GstFlowReturn
gst_dsdetectcolors_transform_ip (GstBaseTransform * btrans, GstBuffer * inbuf)
{
GstDsDetectColors *dsdetectcolors = GST_DSDETECTCOLORS (btrans);
GstMapInfo in_map_info;
GstFlowReturn flow_ret = GST_FLOW_ERROR;
gdouble scale_ratio = 1.0;
DsDetectColorsOutput *output;
NvBufSurface *surface = NULL;
NvDsBatchMeta *batch_meta = NULL;
NvDsFrameMeta *frame_meta = NULL;
NvDsMetaList * l_frame = NULL;
guint i = 0;
dsdetectcolors->frame_num++;
CHECK_CUDA_STATUS (cudaSetDevice (dsdetectcolors->gpu_id),
"Unable to set cuda device");
memset (&in_map_info, 0, sizeof (in_map_info));
if (!gst_buffer_map (inbuf, &in_map_info, GST_MAP_READ)) {
g_print ("Error: Failed to map gst buffer\n");
goto error;
}
nvds_set_input_system_timestamp (inbuf, GST_ELEMENT_NAME (dsdetectcolors));
surface = (NvBufSurface *) in_map_info.data;
GST_DEBUG_OBJECT (dsdetectcolors,
"Processing Frame %" G_GUINT64_FORMAT " Surface %p\n",
dsdetectcolors->frame_num, surface);
if (CHECK_NVDS_MEMORY_AND_GPUID (dsdetectcolors, surface))
goto error;
batch_meta = gst_buffer_get_nvds_batch_meta (inbuf);
if (batch_meta == nullptr) {
GST_ELEMENT_ERROR (dsdetectcolors, STREAM, FAILED,
("NvDsBatchMeta not found for input buffer."), (NULL));
return GST_FLOW_ERROR;
}
// GST_DEBUG_OBJECT (dsdetectcolors,
// "process_full_frame %d\n",
// dsdetectcolors->process_full_frame);
if (dsdetectcolors->process_full_frame) {
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
l_frame = l_frame->next)
{
frame_meta = (NvDsFrameMeta *) (l_frame->data);
NvOSD_RectParams rect_params;
/* Scale the entire frame to processing resolution */
rect_params.left = 0;
rect_params.top = 0;
rect_params.width = dsdetectcolors->video_info.width;
rect_params.height = dsdetectcolors->video_info.height;
/* Process to get the output */
output =
DsDetectColorsProcess (dsdetectcolors->dsdetectcolorslib_ctx,
dsdetectcolors->cvmat->data);
/* Attach the metadata for the full frame */
attach_metadata_full_frame (dsdetectcolors, frame_meta, scale_ratio, output, i);
i++;
free (output);
}
} else {
/* Using object crops as input to the algorithm. The objects are detected by
* the primary detector */
NvDsMetaList * l_obj = NULL;
NvDsObjectMeta *obj_meta = NULL;
if(!dsdetectcolors->is_integrated) {
if (dsdetectcolors->blur_objects) {
if (!(surface->memType == NVBUF_MEM_CUDA_UNIFIED || surface->memType == NVBUF_MEM_CUDA_PINNED)){
GST_ELEMENT_ERROR (dsdetectcolors, STREAM, FAILED,
("%s:need NVBUF_MEM_CUDA_UNIFIED or NVBUF_MEM_CUDA_PINNED memory for opencv blurring",__func__), (NULL));
return GST_FLOW_ERROR;
}
}
}
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
l_frame = l_frame->next)
{
frame_meta = (NvDsFrameMeta *) (l_frame->data);
std::vector<NvDsObjectMeta*> markerObjects;
for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;
l_obj = l_obj->next)
{
obj_meta = (NvDsObjectMeta *) (l_obj->data);
if (strncmp(obj_meta->text_params.display_text,"marker", strlen("marker")) == 0){
markerObjects.push_back(obj_meta);
}
}
//found the cornetr that inside the marker
for (NvDsObjectMeta* obj_meta_marker : markerObjects) {
g_mutex_lock(&lock);
cv::cuda::GpuMat crop;
if (get_converted_mat (dsdetectcolors,
surface, frame_meta->batch_id, &obj_meta_marker->rect_params,
scale_ratio, dsdetectcolors->video_info.width,
dsdetectcolors->video_info.height, crop) != GST_FLOW_OK) {
/* Error in conversion, skip processing on object. */
continue;
}
#ifdef DETECTCOLORS_DEBUG
std::stringstream ss;
ss << "./file" << dsdetectcolors->frame_num << "_3.jpg";
std::string filename = ss.str();
#endif
}
g_mutex_unlock(&lock);
}
}
flow_ret = GST_FLOW_OK;
error:
nvds_set_output_system_timestamp (inbuf, GST_ELEMENT_NAME (dsdetectcolors));
gst_buffer_unmap (inbuf, &in_map_info);
return flow_ret;
}
/**
* Attach metadata for the full frame. We will be adding a new metadata.
*/
static void
attach_metadata_full_frame (GstDsDetectColors * dsdetectcolors, NvDsFrameMeta *frame_meta,
gdouble scale_ratio, DsDetectColorsOutput * output, guint batch_id)
{
NvDsBatchMeta *batch_meta = frame_meta->base_meta.batch_meta;
NvDsObjectMeta *object_meta = NULL;
static gchar font_name[] = "Serif";
//GST_DEBUG_OBJECT (dsdetectcolors, "Attaching metadata %d\n", output->numObjects);
for (gint i = 0; i < output->numObjects; i++) {
DsDetectColorsObject *obj = &output->object[i];
object_meta = nvds_acquire_obj_meta_from_pool(batch_meta);
NvOSD_RectParams & rect_params = object_meta->rect_params;
NvOSD_TextParams & text_params = object_meta->text_params;
/* Assign bounding box coordinates */
rect_params.left = obj->left;
rect_params.top = obj->top;
rect_params.width = obj->width;
rect_params.height = obj->height;
/* Semi-transparent yellow background */
rect_params.has_bg_color = 0;
rect_params.bg_color = (NvOSD_ColorParams) {
1, 1, 0, 0.4};
/* Red border of width 6 */
rect_params.border_width = 3;
rect_params.border_color = (NvOSD_ColorParams) {
1, 0, 0, 1};
/* Scale the bounding boxes proportionally based on how the object/frame was
* scaled during input */
rect_params.left /= scale_ratio;
rect_params.top /= scale_ratio;
rect_params.width /= scale_ratio;
rect_params.height /= scale_ratio;
GST_DEBUG_OBJECT (dsdetectcolors, "Attaching rect%d of batch%u"
" left->%f top->%f width->%f"
" height->%f label->%s\n", i, batch_id, rect_params.left,
rect_params.top, rect_params.width, rect_params.height, obj->label);
object_meta->object_id = UNTRACKED_OBJECT_ID;
g_strlcpy (object_meta->obj_label, obj->label, MAX_LABEL_SIZE);
/* display_text required heap allocated memory */
text_params.display_text = g_strdup (obj->label);
/* Display text above the left top corner of the object */
text_params.x_offset = rect_params.left;
text_params.y_offset = rect_params.top - 10;
/* Set black background for the text */
text_params.set_bg_clr = 1;
text_params.text_bg_clr = (NvOSD_ColorParams) {
0, 0, 0, 1};
/* Font face, size and color */
text_params.font_params.font_name = font_name;
text_params.font_params.font_size = 11;
text_params.font_params.font_color = (NvOSD_ColorParams) {
1, 1, 1, 1};
nvds_add_obj_meta_to_frame(frame_meta, object_meta, NULL);
frame_meta->bInferDone = TRUE;
}
}
/**
* Only update string label in an existing object metadata. No bounding boxes.
* We assume only one label per object is generated
*/
static void
attach_metadata_object (GstDsDetectColors * dsdetectcolors, NvDsObjectMeta * obj_meta,
DsDetectColorsOutput * output)
{
if (output->numObjects == 0)
return;
NvDsBatchMeta *batch_meta = obj_meta->base_meta.batch_meta;
NvDsClassifierMeta *classifier_meta =
nvds_acquire_classifier_meta_from_pool (batch_meta);
classifier_meta->unique_component_id = dsdetectcolors->unique_id;
NvDsLabelInfo *label_info =
nvds_acquire_label_info_meta_from_pool (batch_meta);
g_strlcpy (label_info->result_label, output->object[0].label, MAX_LABEL_SIZE);
nvds_add_label_info_meta_to_classifier(classifier_meta, label_info);
nvds_add_classifier_meta_to_object (obj_meta, classifier_meta);
nvds_acquire_meta_lock (batch_meta);
NvOSD_TextParams & text_params = obj_meta->text_params;
NvOSD_RectParams & rect_params = obj_meta->rect_params;
/* Below code to display the result */
/* Set black background for the text
* display_text required heap allocated memory */
if (text_params.display_text) {
gchar *conc_string = g_strconcat (text_params.display_text, " ",
output->object[0].label, NULL);
g_free (text_params.display_text);
text_params.display_text = conc_string;
} else {
/* Display text above the left top corner of the object */
text_params.x_offset = rect_params.left;
text_params.y_offset = rect_params.top - 10;
text_params.display_text = g_strdup (output->object[0].label);
/* Font face, size and color */
text_params.font_params.font_name = (char *)"Serif";
text_params.font_params.font_size = 11;
text_params.font_params.font_color = (NvOSD_ColorParams) {
1, 1, 1, 1};
/* Set black background for the text */
text_params.set_bg_clr = 1;
text_params.text_bg_clr = (NvOSD_ColorParams) {
0, 0, 0, 1};
}
nvds_release_meta_lock (batch_meta);
}
/**
* Boiler plate for registering a plugin and an element.
*/
static gboolean
dsdetectcolors_plugin_init (GstPlugin * plugin)
{
GST_DEBUG_CATEGORY_INIT (gst_dsdetectcolors_debug, "dsdetectcolors", 0,
"dsdetectcolors plugin");
return gst_element_register (plugin, "dsdetectcolors", GST_RANK_PRIMARY,
GST_TYPE_DSDETECTCOLORS);
}
GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
GST_VERSION_MINOR,
nvdsgst_dsdetectcolors,
DESCRIPTION, dsdetectcolors_plugin_init, DS_VERSION, LICENSE, BINARY_PACKAGE, URL)
ere
gstdsdetectcolors.h
/**
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef __GST_DSDETECTCOLORS_H__
#define __GST_DSDETECTCOLORS_H__
#include <gst/base/gstbasetransform.h>
#include <gst/video/video.h>
/* Open CV headers */
#pragma GCC diagnostic push
#if __GNUC__ >= 8
#pragma GCC diagnostic ignored "-Wclass-memaccess"
#endif
#pragma GCC diagnostic pop
#include <cuda.h>
#include <cuda_runtime.h>
#include "nvbufsurface.h"
#include "nvbufsurftransform.h"
#include "gst-nvquery.h"
#include "gstnvdsmeta.h"
#include "dsdetectcolors_lib/dsdetectcolors_lib.h"
#include "cudaEGL.h"
#ifdef WITH_OPENCV
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/highgui/highgui.hpp"
#include <opencv2/cudafilters.hpp>
#include "opencv2/core.hpp"
#include "opencv2/opencv.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/videoio.hpp"
#include "opencv2/core/cuda.hpp"
#include "opencv2/cudaimgproc.hpp"
#endif
/* Package and library details required for plugin_init */
#define PACKAGE "dsdetectcolors"
#define VERSION "1.0"
#define LICENSE "Proprietary"
#define DESCRIPTION "NVIDIA example plugin for integration with DeepStream on DGPU"
#define BINARY_PACKAGE "NVIDIA DeepStream 3rdparty IP integration example plugin"
#define URL "http://nvidia.com/"
G_BEGIN_DECLS
/* Standard boilerplate stuff */
typedef struct _GstDsDetectColors GstDsDetectColors;
typedef struct _GstDsDetectColorsClass GstDsDetectColorsClass;
/* Standard boilerplate stuff */
#define GST_TYPE_DSDETECTCOLORS (gst_dsdetectcolors_get_type())
#define GST_DSDETECTCOLORS(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_DSDETECTCOLORS,GstDsDetectColors))
#define GST_DSDETECTCOLORS_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_DSDETECTCOLORS,GstDsDetectColorsClass))
#define GST_DSDETECTCOLORS_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_DSDETECTCOLORS, GstDsDetectColorsClass))
#define GST_IS_DSDETECTCOLORS(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_DSDETECTCOLORS))
#define GST_IS_DSDETECTCOLORS_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_DSDETECTCOLORS))
#define GST_DSDETECTCOLORS_CAST(obj) ((GstDsDetectColors *)(obj))
struct _GstDsDetectColors
{
GstBaseTransform base_trans;
// Context of the custom algorithm library
DsDetectColorsCtx *dsdetectcolorslib_ctx;
// Unique ID of the element. The labels generated by the element will be
// updated at index `unique_id` of attr_info array in NvDsObjectParams.
guint unique_id;
// Frame number of the current input buffer
guint64 frame_num;
// CUDA Stream used for allocating the CUDA task
cudaStream_t cuda_stream;
// Host buffer to store RGB data for use by algorithm
void *host_rgb_buf;
// the intermediate scratch buffer for conversions RGBA
NvBufSurface *inter_buf;
#ifdef WITH_OPENCV
// OpenCV mat containing RGB data
cv::cuda::GpuMat *cvmat;
#endif
// Input video info (resolution, color format, framerate, etc)
GstVideoInfo video_info;
// Resolution at which frames/objects should be processed
gint processing_width;
gint processing_height;
// Flag which defince igpu/dgpu
guint is_integrated;
// Amount of objects processed in single call to algorithm
guint batch_size;
// GPU ID on which we expect to execute the task
guint gpu_id;
// GPU ID on which we expect to execute the task
guint meta_id;
// Boolean indicating if entire frame or cropped objects should be processed
gboolean process_full_frame;
// Boolean indicating if to blur the detected objects
gboolean blur_objects;
};
// Boiler plate stuff
struct _GstDsDetectColorsClass
{
GstBaseTransformClass parent_class;
};
GType gst_dsdetectcolors_get_type (void);
G_END_DECLS
#endif /* __GST_DSDETECTCOLORS_H__ */
Thanks
