Please provide complete information as applicable to your setup.
• Hardware Platform : GPU
• DeepStream Version : 6.1.1
• TensorRT Version : 8.4.1.5
• NVIDIA GPU Driver Version (valid for GPU only) : 525.60.13
• Issue Type : bugs
• How to reproduce the issue ? (This is for bugs. Including which sample app is using, the configuration files content, the command line used and other details for reproducing)
I have Yolov8 finetuned for person detection running on triton server. NMS is baked inside the model itself. Its running succesfully on ports 8000, 8001 and 8002.
I am using pyds and making grpc calls to this server from this config
infer_config {
unique_id: 1
gpu_ids: [0]
max_batch_size: 16
backend {
triton {
model_name: “yolov8_nms_tensorrt”
version: -1
grpc {
url: “127.0.0.1:8001”
enable_cuda_buffer_sharing: true
}
}
}preprocess {
network_format: MEDIA_FORMAT_NONE
tensor_order: TENSOR_ORDER_LINEAR
tensor_name: “images”
maintain_aspect_ratio: 0
frame_scaling_hw: FRAME_SCALING_HW_DEFAULT
frame_scaling_filter: 1
normalize {
scale_factor: 0.0039215697906911373
channel_offsets: [0, 0, 0]
}
}
custom_lib {
path : “/opt/nvidia/deepstream/deepstream-6.1/people-app/configs/libnvdsinfer_custom_bbox_yoloV8.so”
}
postprocess {
labelfile_path: “labels_yolov8.txt”
detection {
num_detected_classes: 80
per_class_params {
key: 0
value { pre_threshold: 0.4 }
}
custom_parse_bbox_func:“NvDsInferParseCustomYoloV8”
}
}extra {
copy_input_to_host_buffers: false
output_buffer_pool_size: 2
}
}
input_control {
process_mode: PROCESS_MODE_FULL_FRAME
operate_on_gie_id: -1
interval: 0
}
I have written the following C++ code to convert bboxes from yolov8 format to deepstream format
#include "nvdsinfer_custom_impl.h"
#include <cassert>
#include <iostream>
using namespace std;
/**
* Function expected by DeepStream for decoding the TinyYOLOv2 output.
*
* C-linkage [extern "C"] was written to prevent name-mangling. This function must return true after
* adding all bounding boxes to the objectList vector.
*
* @param [outputLayersInfo] std::vector of NvDsInferLayerInfo objects with information about the output layer.
* @param [networkInfo] NvDsInferNetworkInfo object with information about the TinyYOLOv2 network.
* @param [detectionParams] NvDsInferParseDetectionParams with information about some config params.
* @param [objectList] std::vector of NvDsInferParseObjectInfo objects to which bounding box information must
* be stored.
*
* @return true
*/
// This is just the function prototype. The definition is written at the end of the file.
extern "C" bool NvDsInferParseCustomYoloV8(
std::vector<NvDsInferLayerInfo> const& outputLayersInfo,
NvDsInferNetworkInfo const& networkInfo,
NvDsInferParseDetectionParams const& detectionParams,
std::vector<NvDsInferParseObjectInfo>& objectList);
static __inline__ float bbox_clip(const float& val, const float& minVal = 0.f, const float& maxVal = 1280.f)
{
assert(minVal <= maxVal);
return std::max(std::min(val, (maxVal - 1)), minVal);
}
static std::vector<NvDsInferParseObjectInfo> decodeYoloV8Tensor(
const int* num_dets,
const float* bboxes,
const float* scores,
const int* labels,
const unsigned int& img_w,
const unsigned int& img_h
)
{
cout << "decodeYoloV8Tensor called" << "\n";
std::vector<NvDsInferParseObjectInfo> bboxInfo;
size_t nums = num_dets[0];
for (size_t i = 0; i < nums; i++)
{
float x0 = (bboxes[i * 4]);
float y0 = (bboxes[i * 4 + 1]);
float x1 = (bboxes[i * 4 + 2]);
float y1 = (bboxes[i * 4 + 3]);
x0 = bbox_clip(x0, 0.f, img_w);
y0 = bbox_clip(y0, 0.f, img_h);
x1 = bbox_clip(x1, 0.f, img_w);
y1 = bbox_clip(y1, 0.f, img_h);
NvDsInferParseObjectInfo obj;
obj.left = x0;
obj.top = y0;
obj.width = x1 - x0;
obj.height = y1 - y0;
obj.detectionConfidence = scores[i];
obj.classId = labels[i];
// if (obj.detectionConfidence > 0.4)
cout << "COORD FROM C++ -------------->" << obj.left << " " << obj.top << " "<< obj.width << " " << obj.height << " " << obj.detectionConfidence<< " " << obj.classId<< endl;
bboxInfo.push_back(obj);
}
return bboxInfo;
}
/* C-linkage to prevent name-mangling */
extern "C" bool NvDsInferParseCustomYoloV8(
std::vector<NvDsInferLayerInfo> const& outputLayersInfo,
NvDsInferNetworkInfo const& networkInfo,
NvDsInferParseDetectionParams const& detectionParams,
std::vector<NvDsInferParseObjectInfo>& objectList)
{
// Some assertions and error checking.
// for (long unsigned int i=0; i<outputLayersInfo.size(); i++) {
// cout << outputLayersInfo[i].layerName;
// }
// cout << "\n";
cout << "NvDsInferParseCustomYoloV8 called" << "\n";
if (outputLayersInfo.empty() || outputLayersInfo.size() != 4)
{
std::cerr << "Could not find output layer in bbox parsing" << std::endl;
return false;
}
// Obtaining the output layer.
const NvDsInferLayerInfo& bboxes = outputLayersInfo[0];
const NvDsInferLayerInfo& labels = outputLayersInfo[1];
const NvDsInferLayerInfo& num_dets = outputLayersInfo[2];
const NvDsInferLayerInfo& scores = outputLayersInfo[3];
// num_dets(int) bboxes(float) scores(float) labels(int)
assert (num_dets.dims.numDims == 2);
assert (bboxes.dims.numDims == 3);
assert (scores.dims.numDims == 2);
assert (labels.dims.numDims == 2);
// Decoding the output tensor of YOLOv8 to the NvDsInferParseObjectInfo format.
std::vector<NvDsInferParseObjectInfo> objects =
decodeYoloV8Tensor(
(const int*)(num_dets.buffer),
(const float*)(bboxes.buffer),
(const float*)(scores.buffer),
(const int*)(labels.buffer),
networkInfo.width,
networkInfo.height
);
objectList.clear();
objectList = objects;
return true;
}
/* Check that the custom function has been defined correctly */
CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomYoloV8);
For all frames cout << "COORD FROM C++ -------------->" << obj.left << " " << obj.top << " "<< obj.width << " " << obj.height << " " << obj.detectionConfidence<< " " << obj.classId<< endl;
is printed correctly. For some frames l_obj(in python probe below) has the correct coordinates but for some frames it becomes None i.e obj_meta_list does not get data
def tiler_sink_pad_buffer_probe(pad,info,u_data):
frame_number=0
num_rects=0
gst_buffer = info.get_buffer()
if not gst_buffer:
print("Unable to get GstBuffer ")
return
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
l_frame = batch_meta.frame_meta_list
while l_frame is not None:
try:
frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
n_frame = pyds.get_nvds_buf_surface(hash(gst_buffer), frame_meta.batch_id)
frame_copy = np.array(n_frame, copy=True, order='C')
frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_RGBA2BGRA)
except StopIteration:
break
frame_number=frame_meta.frame_num
l_obj=frame_meta.obj_meta_list
What could be the reason for this?