Please provide complete information as applicable to your setup.
• Geforce RTX 4070 Ti Laptop Gpu
• DeepStream Version 8.0
• Docker Container nvcr.io/nvidia/deepstream:8.0-triton-multiarch
• NVIDIA GPU Driver Version 575.64.03
• Issue Type BUG
• How to reproduce the issue ?
Pipeline structure: source → pgie → postprocessor → preprocessor → sgie → postprocessor
With this structure the last postprocessor has no way to access the raw output tensor. Reason is, that when sgie is configured with:
output-tensor-meta=1
input-tensor-from-meta=1
process-mode=2
network-type=100
the function attach_tensor_output_meta of gstnvinfer_meta_utils.cpp is NOT attaching the metadata to either object or frame meta, only to roi meta. The output can only be accessed through the user_meta_pool of the batch_meta object, but using this workaround there is no way to get the actual object the NvDsInferTensorMeta object belongs to, only the correct gie uid can be matched, but not the actual input object/roi for this output.
Following is my output_thread of postprocessor, which works completeley fine for pgie but not for sgie. For activated preprocessor, i added only prints to see if any data is available as stated by https://docs.nvidia.com/metropolis/deepstream/8.0/text/DS_plugin_gst-nvinfer.html#tensor-metadata docs, but there is no data attached either to NVDS_ROI_METAs roi_userdata_list (Which is ACTIVELY set to null in the meta utils function i mentioned) userdata nor directly to obj_user_meta_list and frame_user_meta_list:
Without the preprocessor in the pipeline, my pipeline works as expected, exept for bad detections caused by missing preprocessing
void InstanceSegImpl::output_thread(void) {
GstFlowReturn flow_ret;
GstBuffer *outBuffer = NULL;
std::unique_lock<std::mutex> lk(m_process_lock);
NvDsBatchMeta *batch_meta = NULL;
int32_t frame_cnt = 0;
while (1) {
if (m_process_q.empty()) {
if (m_stop) {
break;
}
m_process_cv.wait(lk);
continue;
}
PacketInfo packetInfo = m_process_q.front();
m_process_q.pop();
m_process_cv.notify_all();
lk.unlock();
NvBufSurface *in_surf = get_nvbuf_surface(packetInfo.inbuf);
batch_meta = get_nvds_batch_meta(packetInfo.inbuf);
outBuffer = packetInfo.inbuf;
nvds_set_input_system_timestamp(outBuffer, GST_ELEMENT_NAME(m_element));
if (m_preprocessor_support) {
for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) {
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)l_frame->data;
/* First try: Search in ROI metadata (original approach) */
for (NvDsMetaList *l_user = frame_meta->frame_user_meta_list; l_user != NULL; l_user = l_user->next) {
NvDsUserMeta *roi_user_meta = (NvDsUserMeta *)l_user->data;
if (roi_user_meta->base_meta.meta_type == NVDS_ROI_META) {
/* convert to roi metadata */
NvDsRoiMeta *roi_meta = (NvDsRoiMeta *)roi_user_meta->user_meta_data;
printf("frame_usermeta roi list: %p\n", roi_meta->roi_user_meta_list);
} else if (roi_user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) {
/* convert to roi metadata */
NvDsInferTensorMeta *tensor_meta = (NvDsInferTensorMeta *)roi_user_meta->user_meta_data;
printf("frame_usermeta tensor gie uid: %d\n", tensor_meta->unique_id);
} else {
printf("roi_user_meta hat typ: %d\n", roi_user_meta->base_meta.meta_type);
}
}
for (NvDsMetaList *l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) {
NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data;
for (NvDsMetaList *l_user = obj_meta->obj_user_meta_list; l_user != NULL; l_user = l_user->next) {
NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data;
if (user_meta->base_meta.meta_type == NVDS_ROI_META) {
NvDsRoiMeta *roi_meta = (NvDsRoiMeta *)user_meta->user_meta_data;
printf("object_usermeta roi list: %p\n", roi_meta->roi_user_meta_list);
} else if (user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) {
/* convert to roi metadata */
NvDsInferTensorMeta *tensor_meta = (NvDsInferTensorMeta *)user_meta->user_meta_data;
printf("frame_usermeta tensor gie uid: %d\n", tensor_meta->unique_id);
} else {
printf("roi_user_meta hat typ: %d\n", user_meta->base_meta.meta_type);
}
}
}
}
} else {
/* Iterate each frame metadata in batch */
for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) {
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)l_frame->data;
if (m_post_processor_params.process_mode == PROCESS_MODEL_FULL_FRAME) {
/* Iterate user metadata in frames to search PGIE's tensor metadata */
for (NvDsMetaList *l_user = frame_meta->frame_user_meta_list; l_user != NULL;
l_user = l_user->next) {
NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data;
if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META) continue;
/* convert to tensor metadata */
NvDsInferTensorMeta *meta = (NvDsInferTensorMeta *)user_meta->user_meta_data;
// PGIE and operate on meta->unique_id data only
if (meta->unique_id == static_cast<guint>(m_gie_unique_id)) {
for (guint i = 0; i < meta->num_output_layers; i++) {
NvDsInferLayerInfo *info = &meta->output_layers_info[i];
info->buffer = meta->out_buf_ptrs_host[i];
}
/* Parse output tensor and fill detection results into objectList. */
std::vector<NvDsInferLayerInfo> outputLayersInfo(
meta->output_layers_info, meta->output_layers_info + meta->num_output_layers);
std::vector<ObjectData> output;
if (m_post_processor) {
m_post_processor->set_network_info(meta->network_info);
m_post_processor->parse_each_frame(meta, output);
m_post_processor->attach_object_meta(in_surf, frame_meta->batch_id, batch_meta,
frame_meta, NULL, output);
m_post_processor->release_output(output);
} else {
GST_WARNING_OBJECT(m_element, "Post Processor not initialized for network");
}
}
}
} else if (m_post_processor_params.process_mode == PROCESS_MODEL_OBJECTS) {
for (NvDsMetaList *l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) {
NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data;
/* Iterate user metadata in object to search SGIE's tensor data */
for (NvDsMetaList *l_user = obj_meta->obj_user_meta_list; l_user != NULL;
l_user = l_user->next) {
NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data;
if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META) continue;
/* convert to tensor metadata */
NvDsInferTensorMeta *meta = (NvDsInferTensorMeta *)user_meta->user_meta_data;
if (meta->unique_id == static_cast<guint>(m_gie_unique_id)) {
for (unsigned int i = 0; i < meta->num_output_layers; i++) {
NvDsInferLayerInfo *info = &meta->output_layers_info[i];
info->buffer = meta->out_buf_ptrs_host[i];
}
std::vector<NvDsInferLayerInfo> outputLayersInfo(
meta->output_layers_info, meta->output_layers_info + meta->num_output_layers);
std::vector<ObjectData> output;
if (m_post_processor) {
m_post_processor->set_network_info(meta->network_info);
m_post_processor->parse_each_frame(meta, output);
/* Generate classifer metadata and attach to obj_meta */
m_post_processor->attach_object_meta(in_surf, frame_meta->batch_id, batch_meta,
frame_meta, obj_meta, output);
m_post_processor->release_output(output);
} else {
GST_WARNING_OBJECT(m_element, "Post Processor not initialized for network");
}
}
}
}
}
}
}
nvds_set_output_system_timestamp(outBuffer, GST_ELEMENT_NAME(m_element));
flow_ret = gst_pad_push(GST_BASE_TRANSFORM_SRC_PAD(m_element), outBuffer);
GST_DEBUG_OBJECT(m_element,
"CustomLib: %s in_surf=%p, Pushing Frame %d to downstream... Frame %d flow_ret = %d"
" TS=%" GST_TIME_FORMAT " \n",
__func__, in_surf, packetInfo.frame_num, frame_cnt++, flow_ret,
GST_TIME_ARGS(GST_BUFFER_PTS(outBuffer)));
lk.lock();
continue;
}
m_output_thread_stopped = true;
lk.unlock();
return;
}