Issue Summary
- When using the primary detector provided by Nvidia (
/opt/nvidia/deepstream/deepstream-6.0/samples/models/Primary_Detector/resnet10.caffemodel) the tracker works - When using my custom object detection model integrated with
nvinferserverthe tracker does not return any object. Metadata arrives to the tracker’s sink pad, but no object can be found in the tracker’s source pad.
Pipeline
… → nvstreamdemux → nvinferserver → nvtracker → nvstreamdemux → nvvidconvs → nvods → …
Issue technical
I attached a probe to the source pad of nvinferserver and to the source pad of nvtracker. The probe simply prints many different metadata retrieved from the pipeline. The probe function is the same for both the elements.
The probe attached to the pgie prints something like the one just below. I noted that all the objects have object_id equal to 18446744073709551615 right after the pgie. I guess that’s because I am setting that value of object_id to 0xffffffffffffffff when creating the objects in my custom postprocessing function and print(0xffffffffffffffff) is 18446744073709551615 .
{'class_id': 2,
'classifier_meta_list': [],
'confidence': 0.9045451879501343,
'misc_obj_info': array([0, 0, 0, 0], dtype=int32),
'obj_label': 'car',
'obj_user_meta_list': {},
'object_id': 18446744073709551615,
'parent': None,
'rect_params': {'bg_color': <pyds.NvOSD_ColorParams object at 0x7fddc110bef0>,
'border_color': <pyds.NvOSD_ColorParams object at 0x7fddd2b04570>,
'border_width': 3,
'color_id': 0,
'has_bg_color': 0,
'has_color_info': 0,
'height': 325.0,
'left': 1.0,
'reserved': 0,
'top': 642.0,
'width': 261.0},
'reserved': array([0, 0, 0, 0], dtype=int32),
'text_params': <pyds.NvOSD_TextParams object at 0x7fda94344e30>,
'tracker_confidence': 0.0,
'unique_component_id': 0}
The probe attached to the tracker prints stuff like this, while no objects seem to be attached to the metadata, a pattern like this is repeated:
streamId= 2
surfaceStreamID= 8589934592
streamId= 3
surfaceStreamID= 12884901888
streamId= 4
surfaceStreamID= 17179869184
....
Tracker
The issue does not seem related to the tracker alone, since the tracker works fine with the nvidia nvinfer model.
For simplicity, I am using an IoU tracker:
tracker = Gst.ElementFactory.make("nvtracker", "tracker")
tracker.set_property("tracker-width", 640)
tracker.set_property("tracker-height", 384)
tracker.set_property("gpu-id", 0)
tracker.set_property("ll-lib-file", "/opt/nvidia/deepstream/deepstream-6.0/lib/libnvds_nvmultiobjecttracker.so")
tracker.set_property("ll-config-file", "iou_config.txt")
tracker.set_property("enable-past-frame", 1)
tracker.set_property("enable-batch-process", 1)
iou_config.txt:
[IOU_CONFIG]
iou-threshold=0.1
Probe function
The function is basically a copy paste from the probe function of this example: deepstream_python_apps/deepstream_test_2.py at master · NVIDIA-AI-IOT/deepstream_python_apps · GitHub . For reference you can find the code below.
Note that when attaching the probe to the tracker, frame_meta.obj_meta_list does not contain any object. Also, pyds.NvDsPastFrameObjStream.list(trackobj) does not contain any object.
def osd_sink_pad_buffer_probe(pad, info, u_data):
frame_number = 0
# Intiallizing object counter with 0.
obj_counter = defaultdict(int)
num_rects = 0
gst_buffer = info.get_buffer()
if not gst_buffer:
print("Unable to get GstBuffer ")
return
# Retrieve batch metadata from the gst_buffer
# Note that pyds.gst_buffer_get_nvds_batch_meta() expects the
# C address of gst_buffer as input, which is obtained with hash(gst_buffer)
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
l_frame = batch_meta.frame_meta_list
while l_frame is not None:
try:
# Note that l_frame.data needs a cast to pyds.NvDsFrameMeta
# The casting is done by pyds.NvDsFrameMeta.cast()
# The casting also keeps ownership of the underlying memory
# in the C code, so the Python garbage collector will leave
# it alone.
frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
except StopIteration:
break
frame_number = frame_meta.frame_num
num_rects = frame_meta.num_obj_meta
print("frame", num_rects)
l_obj = frame_meta.obj_meta_list
while l_obj is not None:
try:
# Casting l_obj.data to pyds.NvDsObjectMeta
obj_meta = pyds.NvDsObjectMeta.cast(l_obj.data)
except StopIteration:
break
obj_counter[obj_meta.class_id] += 1 ## <-------- This never happens when the probe is attached to the tracker!!
rect_params = pyds.NvOSD_RectParams.cast(obj_meta.rect_params)
rect_params_dict = dict(
bg_color=rect_params.bg_color,
border_color=rect_params.border_color,
border_width=rect_params.border_width,
color_id=rect_params.color_id,
has_bg_color=rect_params.has_bg_color,
has_color_info=rect_params.has_color_info,
height=rect_params.height,
left=rect_params.left,
reserved=rect_params.reserved,
top=rect_params.top,
width=rect_params.width,
)
# Object metadata from secondary models
l_class = obj_meta.classifier_meta_list
# if l_class is None:
# print("l_class is None")
# else:
# print("l_class is not None")
classifier_meta_list_dict = list()
while l_class is not None:
try:
# Note that l_class.data needs a cast to pyds.NvDsClassifierMeta
# The casting is done by pyds.NvDsClassifierMeta.cast()
# The casting also keeps ownership of the underlying memory
# in the C code, so the Python garbage collector will leave
# it alone.
class_meta = pyds.NvDsClassifierMeta.cast(l_class.data)
print("\tClass meta=", class_meta.num_labels, class_meta.unique_component_id)
l_label = class_meta.label_info_list
while l_label is not None:
try:
label_info = pyds.NvDsLabelInfo.cast(l_label.data)
except StopIteration:
break
print("\t\tLabel info=", label_info.result_label, round(label_info.result_prob, 2))
classifier_meta_list_dict.append(
# TODO: can add stuff to the dict
dict(
result_label=label_info.result_label,
result_prob=label_info.result_prob,
)
)
# label_info.num_classes
# label_info.result_label
# label_info.pResult_label
# label_info.result_class_id
# label_info.label_id
# label_info.result_prob
try:
l_label = l_label.next
except StopIteration:
break
except StopIteration:
break
try:
l_class = l_class.next
except StopIteration:
break
try:
l_obj = l_obj.next
except StopIteration:
break
obj_dict = {
"class_id": obj_meta.class_id,
"classifier_meta_list": classifier_meta_list_dict,
"confidence": obj_meta.confidence,
"misc_obj_info": obj_meta.misc_obj_info,
"obj_label": obj_meta.obj_label,
"obj_user_meta_list": dict(),
"object_id": obj_meta.object_id,
"parent": obj_meta.parent, # todo: cast?
"rect_params": rect_params_dict,
"reserved": obj_meta.reserved,
"text_params": obj_meta.text_params,
"unique_component_id": obj_meta.unique_component_id,
# todo: add tracking information
"tracker_confidence": obj_meta.tracker_confidence,
}
pprint(obj_dict)
# Acquiring a display meta object. The memory ownership remains in
# the C code so downstream plugins can still access it. Otherwise
# the garbage collector will claim it when this probe function exits.
display_meta = pyds.nvds_acquire_display_meta_from_pool(batch_meta)
display_meta.num_labels = 1
py_nvosd_text_params = display_meta.text_params[0]
# Setting display text to be shown on screen
# Note that the pyds module allocates a buffer for the string, and the
# memory will not be claimed by the garbage collector.
# Reading the display_text field here will return the C address of the
# allocated string. Use pyds.get_string() to get the string content.
PGIE_CLASS_ID_VEHICLE = 2
PGIE_CLASS_ID_PERSON = 0
py_nvosd_text_params.display_text = f"Counter {dict(obj_counter)}"
# py_nvosd_text_params.display_text = "Frame Number={} Number of Objects={} Vehicle_count={} Person_count={}".format(
# frame_number, num_rects, obj_counter[PGIE_CLASS_ID_VEHICLE], obj_counter[PGIE_CLASS_ID_PERSON])
# Now set the offsets where the string should appear
py_nvosd_text_params.x_offset = 10
py_nvosd_text_params.y_offset = 12
# Font , font-color and font-size
py_nvosd_text_params.font_params.font_name = "Serif"
py_nvosd_text_params.font_params.font_size = 10
# set(red, green, blue, alpha); set to White
py_nvosd_text_params.font_params.font_color.set(1.0, 1.0, 1.0, 1.0)
# Text background color
py_nvosd_text_params.set_bg_clr = 1
# set(red, green, blue, alpha); set to Black
py_nvosd_text_params.text_bg_clr.set(0.0, 0.0, 0.0, 1.0)
# Using pyds.get_string() to get display_text as string
# print("Text added", pyds.get_string(py_nvosd_text_params.display_text))
pyds.nvds_add_display_meta_to_frame(frame_meta, display_meta)
try:
l_frame = l_frame.next
except StopIteration:
break
# past traking meta data
if True: # (self.past_tracking_meta[0] == 1):
l_user = batch_meta.batch_user_meta_list
while l_user is not None:
try:
# Note that l_user.data needs a cast to pyds.NvDsUserMeta
# The casting is done by pyds.NvDsUserMeta.cast()
# The casting also keeps ownership of the underlying memory
# in the C code, so the Python garbage collector will leave
# it alone
user_meta = pyds.NvDsUserMeta.cast(l_user.data)
except StopIteration:
break
if (user_meta and user_meta.base_meta.meta_type == pyds.NvDsMetaType.NVDS_TRACKER_PAST_FRAME_META):
try:
# Note that user_meta.user_meta_data needs a cast to pyds.NvDsPastFrameObjBatch
# The casting is done by pyds.NvDsPastFrameObjBatch.cast()
# The casting also keeps ownership of the underlying memory
# in the C code, so the Python garbage collector will leave
# it alone
pPastFrameObjBatch = pyds.NvDsPastFrameObjBatch.cast(user_meta.user_meta_data)
except StopIteration:
break
for trackobj in pyds.NvDsPastFrameObjBatch.list(pPastFrameObjBatch):
print("streamId=", trackobj.streamID)
print("surfaceStreamID=", trackobj.surfaceStreamID)
for pastframeobj in pyds.NvDsPastFrameObjStream.list(trackobj):
print("numobj=", pastframeobj.numObj)
print("uniqueId=", pastframeobj.uniqueId)
print("classId=", pastframeobj.classId)
print("objLabel=", pastframeobj.objLabel)
for objlist in pyds.NvDsPastFrameObjList.list(pastframeobj):
print('frameNum:', objlist.frameNum)
print('tBbox.left:', objlist.tBbox.left)
print('tBbox.width:', objlist.tBbox.width)
print('tBbox.top:', objlist.tBbox.top)
print('tBbox.right:', objlist.tBbox.height)
print('confidence:', objlist.confidence)
print('age:', objlist.age)
try:
l_user = l_user.next
except StopIteration:
break
return Gst.PadProbeReturn.OK
My pgie postprocessing function
The function is basically a copy-paste of the function add_obj_meta_to_frame from the example deepstream_python_apps/deepstream_ssd_parser.py at master · NVIDIA-AI-IOT/deepstream_python_apps · GitHub .
UNTRACKED_OBJECT_ID = 0xffffffffffffffff
def add_obj_meta_to_frame(frame_object, batch_meta, frame_meta, label_names):
""" Inserts an object into the metadata """
# this is a good place to insert objects into the metadata.
# Here's an example of inserting a single object.
obj_meta = pyds.nvds_acquire_obj_meta_from_pool(batch_meta)
# Set bbox properties. These are in input resolution.
rect_params = obj_meta.rect_params
rect_params.left = frame_object.left
rect_params.top = frame_object.top
rect_params.width = frame_object.width
rect_params.height = frame_object.height
# Semi-transparent yellow backgroud
rect_params.has_bg_color = 0
rect_params.bg_color.set(1, 1, 0, 0.4)
# Red border of width 3
rect_params.border_width = 3
rect_params.border_color.set(1, 0, 0, 1)
# Set object info including class, detection confidence, etc.
obj_meta.confidence = frame_object.detectionConfidence
obj_meta.class_id = frame_object.classId
# There is no tracking ID upon detection. The tracker will
# assign an ID.
obj_meta.object_id = UNTRACKED_OBJECT_ID
lbl_id = frame_object.classId
if lbl_id >= len(label_names):
lbl_id = 0
# Set the object classification label.
obj_meta.obj_label = label_names[lbl_id]
# Set display text for the object.
txt_params = obj_meta.text_params
if txt_params.display_text:
pyds.free_buffer(txt_params.display_text)
# print("rect_params", int(rect_params.left))
txt_params.x_offset = int(rect_params.left)
txt_params.y_offset = max(0, int(rect_params.top) - 10)
txt_params.display_text = (
f"{label_names[lbl_id]} {frame_object.detectionConfidence:.2f} "
# f"{frame_object.left:.0f} {frame_object.top:.0f} {frame_object.height:.0f} {frame_object.width:.0f}"
)
# Font , font-color and font-size
txt_params.font_params.font_name = "Serif"
txt_params.font_params.font_size = 10
# set(red, green, blue, alpha); set to White
txt_params.font_params.font_color.set(1.0, 1.0, 1.0, 1.0)
# Text background color
txt_params.set_bg_clr = 1
# set(red, green, blue, alpha); set to Black
txt_params.text_bg_clr.set(0.0, 0.0, 0.0, 1.0)
# Insert the object into current frame meta
# This object has no parent
pyds.nvds_add_obj_meta_to_frame(frame_meta, obj_meta, None)
EDIT: I attached a probe to the tracker’s sink pad. I can see objects with metadata there (see below). However, I also attached the same probe to the tracker’s source pad and it can’t find any objects, even in past frames data.
Example of data in the tracker’s sink pad formatted as dict:
{'class_id': 2,
'classifier_meta_list': [],
'confidence': 0.6920480132102966,
'detector_bbox_info': {'height': 0.0, 'left': 0.0, 'top': 0.0, 'width': 0.0},
'misc_obj_info': array([0, 0, 0, 0], dtype=int32),
'obj_label': 'car',
'obj_user_meta_list': {},
'object_id': 18446744073709551615,
'parent': None,
'rect_params': {'bg_color': <pyds.NvOSD_ColorParams object at 0x7fb1190d31f0>,
'border_color': <pyds.NvOSD_ColorParams object at 0x7fb1190d37b0>,
'border_width': 3,
'color_id': 0,
'has_bg_color': 0,
'has_color_info': 0,
'height': 41.0,
'left': 763.0,
'reserved': 0,
'top': 672.0,
'width': 53.0},
'reserved': array([0, 0, 0, 0], dtype=int32),
'text_params': <pyds.NvOSD_TextParams object at 0x7fb1190ccc70>,
'tracker_bbox_info': {'height': 0.0, 'left': 0.0, 'top': 0.0, 'width': 0.0},
'tracker_confidence': 0.0,
'unique_component_id': 1}