After looking at Not able to access raw tensor output as metadata - #10 by theelitepro0224
I also looked at the source code for Deepstrem 8.0 and 7.1 in comparison the newest version is not copying the data correctly i think, the 7.1 version is correctly copying everything to the NvDsUserMetas data.
Deepstream 8.0:
/* Attaches the raw tensor output to the GstBuffer as metadata. */
void attach_tensor_output_meta(GstNvInfer *nvinfer,
GstMiniObject *tensor_out_object,
GstNvInferBatch *batch,
NvDsInferContextBatchOutput *batch_output) {
NvDsBatchMeta *batch_meta =
(nvinfer->process_full_frame || nvinfer->input_tensor_from_meta)
? batch->frames[0].frame_meta->base_meta.batch_meta
: batch->frames[0].obj_meta->base_meta.batch_meta;
/* Create and attach NvDsInferTensorMeta for each frame/object. Also
* increment the refcount of GstNvInferTensorOutputObject. */
for (size_t j = 0; j < batch->frames.size(); j++) {
GstNvInferFrame &frame = batch->frames[j];
/* Processing on ROIs (not frames or objects) skip attaching tensor output
* to frames or objects. */
NvDsInferTensorMeta *meta = new NvDsInferTensorMeta;
meta->unique_id = nvinfer->unique_id;
meta->num_output_layers = nvinfer->output_layers_info->size();
meta->output_layers_info = (NvDsInferLayerInfo *)g_memdup2(
nvinfer->output_layers_info->data(),
meta->num_output_layers * sizeof(NvDsInferLayerInfo));
meta->out_buf_ptrs_host = new void *[meta->num_output_layers];
meta->out_buf_ptrs_dev = new void *[meta->num_output_layers];
meta->gpu_id = nvinfer->gpu_id;
meta->priv_data = gst_mini_object_ref(tensor_out_object);
meta->network_info = nvinfer->network_info;
meta->maintain_aspect_ratio = nvinfer->maintain_aspect_ratio;
meta->symmetric_padding = nvinfer->symmetric_padding;
for (unsigned int i = 0; i < meta->num_output_layers; i++) {
NvDsInferLayerInfo &info = meta->output_layers_info[i];
meta->out_buf_ptrs_dev[i] =
(uint8_t *)batch_output->outputDeviceBuffers[i] +
info.inferDims.numElements * get_element_size(info.dataType) * j;
meta->out_buf_ptrs_host[i] =
(uint8_t *)batch_output->hostBuffers[info.bindingIndex] +
info.inferDims.numElements * get_element_size(info.dataType) * j;
}
NvDsUserMeta *user_meta = nvds_acquire_user_meta_from_pool(batch_meta);
user_meta->user_meta_data = meta;
user_meta->base_meta.meta_type = (NvDsMetaType)NVDSINFER_TENSOR_OUTPUT_META;
user_meta->base_meta.release_func = release_tensor_output_meta;
user_meta->base_meta.copy_func = copy_tensor_output_meta;
user_meta->base_meta.batch_meta = batch_meta;
if (nvinfer->input_tensor_from_meta) {
/* Attach tensor meta as part of ROI Meta */
nvds_add_user_meta_to_roi(frame.roi_meta, user_meta);
/* Create a new user meta for ROI meta at frame level */
NvDsUserMeta *roi_user_meta =
nvds_acquire_user_meta_from_pool(batch_meta);
NvDsRoiMeta *frame_roi_meta = new NvDsRoiMeta;
// Copy ROI data
// WHY DO WE NEED TO COPY THE ROI DATA LIKE THIS ???
frame_roi_meta->roi = frame.roi_meta->roi;
memcpy(frame_roi_meta->roi_polygon, frame.roi_meta->roi_polygon,
sizeof(guint) * DS_MAX_POLYGON_POINTS * 2);
frame_roi_meta->converted_buffer = frame.roi_meta->converted_buffer;
frame_roi_meta->frame_meta = frame.roi_meta->frame_meta;
frame_roi_meta->scale_ratio_x = frame.roi_meta->scale_ratio_x;
frame_roi_meta->scale_ratio_y = frame.roi_meta->scale_ratio_y;
frame_roi_meta->offset_left = frame.roi_meta->offset_left;
frame_roi_meta->offset_top = frame.roi_meta->offset_top;
frame_roi_meta->object_meta = nullptr;
frame_roi_meta->classifier_meta_list = nullptr;
frame_roi_meta->roi_user_meta_list = nullptr;
// This should be replaced as in Deepstrem 7.1
// roi_user_meta->user_meta_data = frame.roi_meta;
roi_user_meta->user_meta_data = frame_roi_meta;
roi_user_meta->base_meta.meta_type = (NvDsMetaType)NVDS_ROI_META;
roi_user_meta->base_meta.release_func = release_user_meta_at_frame_level;
roi_user_meta->base_meta.copy_func = copy_roi_meta;
roi_user_meta->base_meta.batch_meta = batch_meta;
/* Add ROI meta to frame meta */
nvds_add_user_meta_to_frame(frame.frame_meta, roi_user_meta);
/* If object is ROI itself, add ROI meta to object meta */
if (frame.obj_meta) {
// This should be replaced as in Deepstrem 7.1
// obj_roi_user_meta->user_meta_data = frame.roi_meta;
NvDsUserMeta *obj_roi_user_meta =
nvds_acquire_user_meta_from_pool(batch_meta);
NvDsRoiMeta *obj_roi_meta = new NvDsRoiMeta;
// Copy ROI data
obj_roi_meta->roi = frame.roi_meta->roi;
memcpy(obj_roi_meta->roi_polygon, frame.roi_meta->roi_polygon,
sizeof(guint) * DS_MAX_POLYGON_POINTS * 2);
obj_roi_meta->converted_buffer = frame.roi_meta->converted_buffer;
obj_roi_meta->frame_meta = frame.roi_meta->frame_meta;
obj_roi_meta->scale_ratio_x = frame.roi_meta->scale_ratio_x;
obj_roi_meta->scale_ratio_y = frame.roi_meta->scale_ratio_y;
obj_roi_meta->offset_left = frame.roi_meta->offset_left;
obj_roi_meta->offset_top = frame.roi_meta->offset_top;
obj_roi_meta->object_meta = nullptr;
obj_roi_meta->classifier_meta_list = nullptr;
obj_roi_meta->roi_user_meta_list = nullptr;
obj_roi_user_meta->user_meta_data = obj_roi_meta;
obj_roi_user_meta->base_meta.meta_type = (NvDsMetaType)NVDS_ROI_META;
obj_roi_user_meta->base_meta.release_func =
release_user_meta_at_frame_level;
obj_roi_user_meta->base_meta.copy_func = nullptr;
obj_roi_user_meta->base_meta.batch_meta = batch_meta;
nvds_add_user_meta_to_obj(frame.obj_meta, obj_roi_user_meta);
}
} else if (nvinfer->process_full_frame) {
nvds_add_user_meta_to_frame(frame.frame_meta, user_meta);
} else {
nvds_add_user_meta_to_obj(frame.obj_meta, user_meta);
}
}
}
Deepstream 7.1:
/* Attaches the raw tensor output to the GstBuffer as metadata. */
void
attach_tensor_output_meta (GstNvInfer *nvinfer, GstMiniObject * tensor_out_object,
GstNvInferBatch *batch, NvDsInferContextBatchOutput *batch_output)
{
NvDsBatchMeta *batch_meta = (nvinfer->process_full_frame
|| nvinfer->input_tensor_from_meta) ? batch->frames[0].
frame_meta->base_meta.batch_meta : batch->frames[0].obj_meta->base_meta.
batch_meta;
/* Create and attach NvDsInferTensorMeta for each frame/object. Also
* increment the refcount of GstNvInferTensorOutputObject. */
for (size_t j = 0; j < batch->frames.size (); j++) {
GstNvInferFrame & frame = batch->frames[j];
/* Processing on ROIs (not frames or objects) skip attaching tensor output
* to frames or objects. */
NvDsInferTensorMeta *meta = new NvDsInferTensorMeta;
meta->unique_id = nvinfer->unique_id;
meta->num_output_layers = nvinfer->output_layers_info->size ();
meta->output_layers_info = (NvDsInferLayerInfo*)g_memdup2(nvinfer->output_layers_info->data (),
meta->num_output_layers * sizeof (NvDsInferLayerInfo)) ;
meta->out_buf_ptrs_host = new void *[meta->num_output_layers];
meta->out_buf_ptrs_dev = new void *[meta->num_output_layers];
meta->gpu_id = nvinfer->gpu_id;
meta->priv_data = gst_mini_object_ref (tensor_out_object);
meta->network_info = nvinfer->network_info;
meta->maintain_aspect_ratio = nvinfer->maintain_aspect_ratio;
meta->symmetric_padding = nvinfer->symmetric_padding;
for (unsigned int i = 0; i < meta->num_output_layers; i++) {
NvDsInferLayerInfo & info = meta->output_layers_info[i];
meta->out_buf_ptrs_dev[i] =
(uint8_t *) batch_output->outputDeviceBuffers[i] +
info.inferDims.numElements * get_element_size (info.dataType) * j;
meta->out_buf_ptrs_host[i] =
(uint8_t *) batch_output->hostBuffers[info.bindingIndex] +
info.inferDims.numElements * get_element_size (info.dataType) * j;
}
NvDsUserMeta *user_meta = nvds_acquire_user_meta_from_pool (batch_meta);
user_meta->user_meta_data = meta;
user_meta->base_meta.meta_type =
(NvDsMetaType) NVDSINFER_TENSOR_OUTPUT_META;
user_meta->base_meta.release_func = release_tensor_output_meta;
user_meta->base_meta.copy_func = copy_tensor_output_meta;
user_meta->base_meta.batch_meta = batch_meta;
if (nvinfer->input_tensor_from_meta) {
/* Attach tensor meta as part of ROI Meta */
nvds_add_user_meta_to_roi (frame.roi_meta, user_meta);
/* Attach ROI Meta to Frame Meta */
NvDsUserMeta *user_meta = nvds_acquire_user_meta_from_pool (batch_meta);
user_meta->user_meta_data = frame.roi_meta;
user_meta->base_meta.meta_type =
(NvDsMetaType) NVDS_ROI_META;
user_meta->base_meta.release_func = release_user_meta_at_frame_level;
user_meta->base_meta.copy_func = NULL;
user_meta->base_meta.batch_meta = batch_meta;
nvds_add_user_meta_to_frame (frame.frame_meta, user_meta);
/* if object is roi itself */
if (frame.obj_meta) {
nvds_add_user_meta_to_obj (frame.obj_meta, user_meta);
}
} else if (nvinfer->process_full_frame) {
nvds_add_user_meta_to_frame (frame.frame_meta, user_meta);
} else {
nvds_add_user_meta_to_obj (frame.obj_meta, user_meta);
}
}
}
After adapting this function, the release_tensor_output_meta function is called too often and results in a segfault.
After many hours of testing and annoyance i got to this solution, luckily roi_meta of NvDsPreProcessorBatchMeta contains frame_meta and object_meta:
for(NvDsMetaList *l_batch_user_meta = batch_meta->batch_user_meta_list; l_batch_user_meta != NULL; l_batch_user_meta = l_batch_user_meta->next) {
NvDsUserMeta *batch_user_meta = (NvDsUserMeta *)l_batch_user_meta->data;
if(batch_user_meta->base_meta.meta_type != NVDS_PREPROCESS_BATCH_META) continue;
GstNvDsPreProcessBatchMeta *preprocess_batch_meta = (GstNvDsPreProcessBatchMeta *)batch_user_meta->user_meta_data;
for(auto roi_meta : preprocess_batch_meta->roi_vector) {
for(NvDsMetaList *l_roi_user = roi_meta.roi_user_meta_list; l_roi_user != NULL; l_roi_user = l_roi_user->next) {
NvDsUserMeta *roi_user_meta = (NvDsUserMeta *)l_roi_user->data;
if(roi_user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META) continue;
NvDsInferTensorMeta *meta = (NvDsInferTensorMeta *)roi_user_meta->user_meta_data;
NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)roi_meta.object_meta;
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)roi_meta.frame_meta;
if (meta->unique_id == static_cast<guint>(m_gie_unique_id)) {
for (guint i = 0; i < meta->num_output_layers; i++) {
NvDsInferLayerInfo *info = &meta->output_layers_info[i];
info->buffer = meta->out_buf_ptrs_host[i];
}
/* Parse output tensor and fill detection results into objectList. */
std::vector<NvDsInferLayerInfo> outputLayersInfo(
meta->output_layers_info, meta->output_layers_info + meta->num_output_layers);
std::vector<ObjectData> output;
if (m_post_processor) {
m_post_processor->set_network_info(meta->network_info);
m_post_processor->parse_each_frame(meta, output);
m_post_processor->attach_object_meta(in_surf, frame_meta->batch_id, batch_meta,
frame_meta, obj_meta, output);
m_post_processor->release_output(output);
} else {
GST_WARNING_OBJECT(m_element, "Post Processor not initialized for network");
}
}
}
}
}