Adding nvtracker to pipeline but nothing in OSD does not show up in the OSD

nakayonum · December 6, 2023, 9:11am

• Hardware Platform (Jetson / GPU) Jetson Xavier NX
• DeepStream Version 6.0.1
• JetPack Version (valid for Jetson only) 4.6.1
• TensorRT Version 8.2.1
• NVIDIA GPU Driver Version (valid for GPU only)
• Issue Type( questions, new requirements, bugs)
• How to reproduce the issue ? (This is for bugs. Including which sample app is using, the configuration files content, the command line used and other details for reproducing)
• Requirement details( This is for new requirement. Including the module name-for which plugin or for which sample application, the function description)

Hello,
I have a problem with the OSD not displaying the trackingID.
I have based on deepstream_test_1_usb.py with the following changes

Inference engine:
e2pose(https://s3.ap-northeast-2.wasabisys.com/pinto-model-zoo/333_E2Pose/resources_post.tar.gz)
Added Nvtracker to the pipeline

The modified source is attached.

main.py

#!/usr/bin/env python3

import sys
sys.path.append('../')
import gi
gi.require_version('Gst', '1.0')
from gi.repository import GLib, Gst
from common.is_aarch_64 import is_aarch64
from common.bus_call import bus_call
import configparser
import pyds

PGIE_CLASS_ID_VEHICLE = 0
PGIE_CLASS_ID_BICYCLE = 1
PGIE_CLASS_ID_PERSON = 2
PGIE_CLASS_ID_ROADSIGN = 3

TRACKER_CONFIG_FILE = "src/sample-0003/cfg_tracker.txt"

import e2pose
def parse_objects_from_tensor_meta(batch_meta, frame_meta, tensor_meta):
    #Reasoning Result Analysis
    results = e2pose.parse_objects_from_tensor_meta(tensor_meta)
    #skeletal drawing+ tracker.add_obj_meta()
    e2pose.add_display_meta(batch_meta, frame_meta, results)
    

def pose_sink_pad_buffer_probe(pad,info,u_data):
    frame_number=0
    #Intiallizing object counter with 0.
    obj_counter = {
        PGIE_CLASS_ID_VEHICLE:0,
        PGIE_CLASS_ID_PERSON:0,
        PGIE_CLASS_ID_BICYCLE:0,
        PGIE_CLASS_ID_ROADSIGN:0
    }
    num_rects=0

    gst_buffer = info.get_buffer()
    if not gst_buffer:
        print("Unable to get GstBuffer ")
        return

    # Retrieve batch metadata from the gst_buffer
    # Note that pyds.gst_buffer_get_nvds_batch_meta() expects the
    # C address of gst_buffer as input, which is obtained with hash(gst_buffer)
    batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
    l_frame = batch_meta.frame_meta_list
    while l_frame is not None:
        try:
           frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
        except StopIteration:
            break

        l_user = frame_meta.frame_user_meta_list
        while l_user is not None:
            try:
                # Casting l_obj.data to pyds.NvDsObjectMeta
                user_meta=pyds.NvDsUserMeta.cast(l_user.data)
                if user_meta.base_meta.meta_type == pyds.NvDsMetaType.NVDSINFER_TENSOR_OUTPUT_META:
                    tensor_meta=pyds.NvDsInferTensorMeta.cast(user_meta.user_meta_data)
                    parse_objects_from_tensor_meta(batch_meta, frame_meta, tensor_meta)
            except StopIteration:
                break
            try: 
                l_user=l_user.next
            except StopIteration:
                break

        try:
            l_frame=l_frame.next
        except StopIteration:
            break
			
    return Gst.PadProbeReturn.OK	

def osd_sink_pad_buffer_probe(pad, info, u_data):

    buffer = info.get_buffer()
    batch = pyds.gst_buffer_get_nvds_batch_meta(hash(buffer))

    l_frame = batch.frame_meta_list
    while l_frame:
        frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
        l_obj = frame_meta.obj_meta_list
        while l_obj:
            obj_meta = pyds.NvDsObjectMeta.cast(l_obj.data)
            obj_meta.text_params.display_text = "person{}"
            print(f'obj_meta.object_id = {obj_meta.object_id}')
            obj_meta.text_params.set_bg_clr = 1 # Set boolean indicating that text has bg color to true.
            obj_meta.text_params.text_bg_clr.set(0.2, 0.2, 0.2, 0.3) # set(red, green, blue, alpha);
            l_obj = l_obj.next
        l_frame = l_frame.next
    return Gst.PadProbeReturn.OK


def main(args):
    # Check input arguments
    if len(args) != 2:
        sys.stderr.write("usage: %s <v4l2-device-path>\n" % args[0])
        sys.exit(1)

    # Standard GStreamer initialization
    Gst.init(None)

    # Create gstreamer elements
    # Create Pipeline element that will form a connection of other elements
    print("Creating Pipeline \n ")
    pipeline = Gst.Pipeline()

    if not pipeline:
        sys.stderr.write(" Unable to create Pipeline \n")

    # Source element for reading from the file
    print("Creating Source \n ")
    source = Gst.ElementFactory.make("v4l2src", "usb-cam-source")
    if not source:
        sys.stderr.write(" Unable to create Source \n")

    caps_v4l2src = Gst.ElementFactory.make("capsfilter", "v4l2src_caps")
    if not caps_v4l2src:
        sys.stderr.write(" Unable to create v4l2src capsfilter \n")


    print("Creating Video Converter \n")

    # videoconvert to make sure a superset of raw formats are supported
    vidconvsrc = Gst.ElementFactory.make("videoconvert", "convertor_src1")
    if not vidconvsrc:
        sys.stderr.write(" Unable to create videoconvert \n")

    # nvvideoconvert to convert incoming raw buffers to NVMM Mem (NvBufSurface API)
    nvvidconvsrc = Gst.ElementFactory.make("nvvideoconvert", "convertor_src2")
    if not nvvidconvsrc:
        sys.stderr.write(" Unable to create Nvvideoconvert \n")

    caps_vidconvsrc = Gst.ElementFactory.make("capsfilter", "nvmm_caps")
    if not caps_vidconvsrc:
        sys.stderr.write(" Unable to create capsfilter \n")

    # Create nvstreammux instance to form batches from one or more sources.
    streammux = Gst.ElementFactory.make("nvstreammux", "Stream-muxer")
    if not streammux:
        sys.stderr.write(" Unable to create NvStreamMux \n")

    # Use nvinfer to run inferencing on camera's output,
    # behaviour of inferencing is set through config file
    pgie = Gst.ElementFactory.make("nvinfer", "primary-inference")
    if not pgie:
        sys.stderr.write(" Unable to create pgie \n")

    tracker = Gst.ElementFactory.make("nvtracker", "tracker")
    if not tracker:
        sys.stderr.write(" Unable to create tracker \n")

    # Use convertor to convert from NV12 to RGBA as required by nvosd
    nvvidconv = Gst.ElementFactory.make("nvvideoconvert", "convertor")
    if not nvvidconv:
        sys.stderr.write(" Unable to create nvvidconv \n")

    # Create OSD to draw on the converted RGBA buffer
    nvosd = Gst.ElementFactory.make("nvdsosd", "onscreendisplay")

    if not nvosd:
        sys.stderr.write(" Unable to create nvosd \n")

    # Finally render the osd output
    if is_aarch64():
        print("Creating nv3dsink \n")
        sink = Gst.ElementFactory.make("nv3dsink", "nv3d-sink")
        if not sink:
            sys.stderr.write(" Unable to create nv3dsink \n")
    else:
        print("Creating EGLSink \n")
        sink = Gst.ElementFactory.make("nveglglessink", "nvvideo-renderer")
        if not sink:
            sys.stderr.write(" Unable to create egl sink \n")

    print("Playing cam %s " %args[1])
    caps_v4l2src.set_property('caps', Gst.Caps.from_string("video/x-raw, framerate=30/1"))
    #caps_v4l2src.set_property('caps', Gst.Caps.from_string("video/x-raw, width=1280, height=720"))
    caps_vidconvsrc.set_property('caps', Gst.Caps.from_string("video/x-raw(memory:NVMM)"))
    source.set_property('device', args[1])
    streammux.set_property('width', 1280)
    streammux.set_property('height', 720)
    streammux.set_property('batch-size', 1)
    streammux.set_property('batched-push-timeout', 4000000)
    streammux.set_property('enable-padding', "true")
    pgie.set_property('output-tensor-meta', True)
    pgie.set_property('config-file-path', "src/sample-0003/config.txt")
    sink.set_property('sync', False)
    set_tracker_config(TRACKER_CONFIG_FILE, tracker)

    print("Adding elements to Pipeline \n")
    pipeline.add(source)
    pipeline.add(caps_v4l2src)
    pipeline.add(vidconvsrc)
    pipeline.add(nvvidconvsrc)
    pipeline.add(caps_vidconvsrc)
    pipeline.add(streammux)
    pipeline.add(pgie)
    pipeline.add(tracker)
    pipeline.add(nvvidconv)
    pipeline.add(nvosd)
    pipeline.add(sink)

    # we link the elements together
    # v4l2src -> nvvideoconvert -> mux -> 
    # nvinfer -> nvvideoconvert -> nvosd -> video-renderer
    print("Linking elements in the Pipeline \n")
    source.link(caps_v4l2src)
    caps_v4l2src.link(vidconvsrc)
    vidconvsrc.link(nvvidconvsrc)
    nvvidconvsrc.link(caps_vidconvsrc)

    sinkpad = streammux.get_request_pad("sink_0")
    if not sinkpad:
        sys.stderr.write(" Unable to get the sink pad of streammux \n")
    srcpad = caps_vidconvsrc.get_static_pad("src")
    if not srcpad:
        sys.stderr.write(" Unable to get source pad of caps_vidconvsrc \n")
    srcpad.link(sinkpad)
    streammux.link(pgie)
    #pgie.link(nvvidconv)
    pgie.link(tracker)
    tracker.link(nvvidconv)
    nvvidconv.link(nvosd)
    nvosd.link(sink)

    # create an event loop and feed gstreamer bus mesages to it
    loop = GLib.MainLoop()
    bus = pipeline.get_bus()
    bus.add_signal_watch()
    bus.connect ("message", bus_call, loop)

    pgiesrcpad = pgie.get_static_pad("src")
    if not pgiesrcpad:
        sys.stderr.write(" Unable to get src pad of pgie \n")

    pgiesrcpad.add_probe(Gst.PadProbeType.BUFFER, pose_sink_pad_buffer_probe, 0)

    # Lets add probe to get informed of the meta data generated, we add probe to
    # the sink pad of the osd element, since by that time, the buffer would have
    # had got all the metadata.
    osdsinkpad = nvosd.get_static_pad("sink")
    if not osdsinkpad:
        sys.stderr.write(" Unable to get sink pad of nvosd \n")

    osdsinkpad.add_probe(Gst.PadProbeType.BUFFER, osd_sink_pad_buffer_probe, 0)

    # start play back and listen to events
    print("Starting pipeline \n")
    pipeline.set_state(Gst.State.PLAYING)
    try:
        loop.run()
    except:
        pass
    # cleanup
    pipeline.set_state(Gst.State.NULL)

def set_tracker_config(cfg_path, tracker):
    config = configparser.ConfigParser()
    config.read(cfg_path)
    config.sections()

    for key in config['tracker']:
        print('set_tracker_config =>')
        if key == 'tracker-width' :
            tracker_width = config.getint('tracker', key)
            tracker.set_property('tracker-width', tracker_width)
        if key == 'tracker-height' :
            tracker_height = config.getint('tracker', key)
            tracker.set_property('tracker-height', tracker_height)
        if key == 'gpu-id' :
            tracker_gpu_id = config.getint('tracker', key)
            tracker.set_property('gpu_id', tracker_gpu_id)
        if key == 'll-lib-file' :
            tracker_ll_lib_file = config.get('tracker', key)
            tracker.set_property('ll-lib-file', tracker_ll_lib_file)
        if key == 'll-config-file' :
            tracker_ll_config_file = config.get('tracker', key)
            tracker.set_property('ll-config-file', tracker_ll_config_file)
        if key == 'enable-past-frame' :
            tracker_enable_past_frame = config.getint('tracker', key)
            tracker.set_property('enable_past_frame', tracker_enable_past_frame)
        if key == 'tracking-id-reset-mode' :
            tracking_id_reset_mode = config.getint('tracker', key)
            tracker.set_property('tracking-id-reset-mode', tracking_id_reset_mode)
        if key == 'display-tracking-id' :
            display_tracking_id = config.getint('tracker', key)
            tracker.set_property('display-tracking-id', display_tracking_id)


if __name__ == '__main__':
    sys.exit(main(sys.argv))

tracker.py

import pyds

def add_obj_meta(frame_meta, batch_meta, cx_left, cy_top, cx_width, cy_height):
    print('tracker add_obj_meta =>')
    pyds.nvds_acquire_meta_lock(batch_meta)
    new_object = pyds.nvds_acquire_obj_meta_from_pool(batch_meta)
    new_object.unique_component_id = 1
    new_object.class_id = 0
    #new_object.confidence = conf
    new_object.obj_label = 'person'
    new_object.parent = None

    rect_params = new_object.rect_params
    rect_params.left = cx_left
    rect_params.top = cy_top
    rect_params.width = cx_width
    rect_params.height = cy_height
    rect_params.border_width = 3
    rect_params.border_color.set(0, 0, 1, 1.0)
    rect_params.has_bg_color = 1
    rect_params.bg_color.set(0.5, 0.5, 0.5, 0.1)

    print('cx_left, cy_top, cx_width, cy_height')
    print(f'{cx_left}, {cy_top}, {cx_width}, {cy_height}')

    text_params = new_object.text_params
    # text_params.display_text = "person{}: {:.2f}".format(new_object.object_id, conf.squeeze())
    text_params.x_offset = cx_left
    text_params.y_offset = max(cy_top-20, 0)
    # text_params.font_params.font_name = "Serif"
    # text_params.font_params.font_color.set(1.0, 1.0, 1.0, 1.0)
    # text_params.font_params.font_size = 10
    # text_params.set_bg_clr = 1 # Set boolean indicating that text has bg color to true.
    # text_params.text_bg_clr.set(0.2, 0.2, 0.2, 0.3) # set(red, green, blue, alpha);

    # raw_box = new_object.detector_bbox_info.org_bbox_coords
    # raw_box.left = int(box[0])
    # raw_box.top = int(box[1])
    # raw_box.width = int(box[2]-box[0])
    # raw_box.height = int(box[3]-box[1])
    pyds.nvds_add_obj_meta_to_frame(frame_meta, new_object, None)
    pyds.nvds_release_meta_lock(batch_meta)

config.txt

[property]
gpu-id=0
#net-scale-factor=0.0174292
#offsets=123.675;116.28;103.53
model-color-format=0
maintain-aspect-ratio=0
onnx-file=/root/models/e2epose_resnet50_1x3x512x512.onnx
#labelfile-path=labels.txt
batch-size=1
process-mode=1
#model-color-format=0
## 0=FP32, 1=INT8, 2=FP16 mode
network-mode=2
#num-detected-classes=4
interval=0
gie-unique-id=1
model-engine-file=/root/models/e2epose_resnet50_1x3x512x512.onnx_b1_gpu0_fp16.engine
network-type=100
workspace-size=3000

cfg_tracker.txt

[tracker]
tracker-width=640
tracker-height=384
gpu-id=0
ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so
#enable-past-frame=1
enable-batch-process=1
tracking-id-reset-mode=3
display-tracking-id=1

When I run this program, the bone drawing and bounding box can be displayed

Here are my questions.

When I run this program, the OSD does not display anything at the text_params.x_offset and text_params.y_offset positions specified in tracker.py. How can I display them?
In osd_sink_pad_buffer_probe(), the value of obj_meta.object_id is always 0. Why is this?

kesong · December 7, 2023, 1:23am

Can you refer test2 sample which already included nvtracker? https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/blob/master/apps/deepstream-test2/deepstream_test_2.py

nakayonum · December 7, 2023, 2:08am

Thanks for the reply.

I have already checked the test2 sample and it is working!
The test2 sample uses the object detection engine in inference and I want to use the pose estimation engine in this case.
So instead of the same as the test2 sample, I am using nvinfer callback (pose_sink_pad_buffer_probe()) to analyze the inference and add_obj_meta() for tracker to create a new object_meta.
Then I check the obj_meta.object_id that I should get from the tracker in the nvosd callback (osd_sink_pad_buffer_probe()) and it is always 0.

I can’t see any significant difference between the ‘dstest2_tracker_config.txt’ in the test2 sample and the ‘cfg_tracker.txt’ I provided.

Does nvtrcker work or not depending on the inference engine?

kesong · December 7, 2023, 10:48am

Can you share how you add new obj meta in probe()? Do you set frameMeta.bInferDone?

nakayonum · December 7, 2023, 11:42pm

Yes.

Here is how to add a new obj meta to probe()

pose_sink_pad_buffer_probe()->parse_objects_from_tensor_meta()->e2pose.add_display_meta()->tracker.add_obj_meta()

def add_obj_meta(frame_meta, batch_meta, cx_left, cy_top, cx_width, cy_height):
    print('tracker add_obj_meta =>')
    pyds.nvds_acquire_meta_lock(batch_meta)
    new_object = pyds.nvds_acquire_obj_meta_from_pool(batch_meta)
    new_object.unique_component_id = 1
    new_object.class_id = 0
    new_object.obj_label = 'person'
    new_object.parent = None

    rect_params = new_object.rect_params
    rect_params.left = cx_left
    rect_params.top = cy_top
    rect_params.width = cx_width
    rect_params.height = cy_height
    rect_params.border_width = 3
    rect_params.border_color.set(0, 0, 1, 1.0)
    rect_params.has_bg_color = 1
    rect_params.bg_color.set(0.5, 0.5, 0.5, 0.1)

    print('cx_left, cy_top, cx_width, cy_height')
    print(f'{cx_left}, {cy_top}, {cx_width}, {cy_height}')

    text_params = new_object.text_params
    text_params.x_offset = cx_left
    text_params.y_offset = max(cy_top-20, 0)

    pyds.nvds_add_obj_meta_to_frame(frame_meta, new_object, None)
    pyds.nvds_release_meta_lock(batch_meta)

This is done for the number of people obtained by inference.
For example, if there are two people, it is done twice.

FrameMeta.bInferDone is not set, so the default value is used.

nakayonum · December 8, 2023, 4:52am

Hi,
Resolved.
Changed pose_sink_pad_buffer_probe() as follows
‘’’
def pose_sink_pad_buffer_probe(pad,info,u_data):
frame_number=0
#Intiallizing object counter with 0.
obj_counter = {
PGIE_CLASS_ID_VEHICLE:0,
PGIE_CLASS_ID_PERSON:0,
PGIE_CLASS_ID_BICYCLE:0,
PGIE_CLASS_ID_ROADSIGN:0
}
num_rects=0

gst_buffer = info.get_buffer()
if not gst_buffer:
    print("Unable to get GstBuffer ")
    return

# Retrieve batch metadata from the gst_buffer
# Note that pyds.gst_buffer_get_nvds_batch_meta() expects the
# C address of gst_buffer as input, which is obtained with hash(gst_buffer)
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
l_frame = batch_meta.frame_meta_list
while l_frame is not None:
    try:
        # Note that l_frame.data needs a cast to pyds.NvDsFrameMeta
        # The casting is done by pyds.NvDsFrameMeta.cast()
        # The casting also keeps ownership of the underlying memory
        # in the C code, so the Python garbage collector will leave
        # it alone.
       frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
    except StopIteration:
        break

    l_user = frame_meta.frame_user_meta_list
    while l_user is not None:
        try:
            # Casting l_obj.data to pyds.NvDsObjectMeta
            user_meta=pyds.NvDsUserMeta.cast(l_user.data)
            if user_meta.base_meta.meta_type == pyds.NvDsMetaType.NVDSINFER_TENSOR_OUTPUT_META:
                tensor_meta=pyds.NvDsInferTensorMeta.cast(user_meta.user_meta_data)
                #print(f'dir(tensor_meta)={dir(tensor_meta)}')
                #print(f'dir(tensor_meta.out_buf_ptrs_host)={dir(tensor_meta.out_buf_ptrs_host)}')
                #print(f'tensor_meta.num_output_layers={tensor_meta.num_output_layers}')
                parse_objects_from_tensor_meta(batch_meta, frame_meta, tensor_meta)
        except StopIteration:
            break
        try: 
            l_user=l_user.next
        except StopIteration:
            break

    try:
+       frame_meta.bInferDone = True
        l_frame=l_frame.next
    except StopIteration:
        break

‘’’
Question.
I got the inference result of posture estimation without “frame_meta.bInferDone = True”. do I need to set bInferDone=True if I use nvtracker?

kesong · December 12, 2023, 10:32am

Yes, you need to set bInferDone=True if I use nvtracker.

system · December 26, 2023, 10:32am

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.

Topic		Replies	Views
Using nvmsgconv and RTSP output on the same DEEPSTREAM pipeline DeepStream SDK rtsp	2	1004	October 12, 2021
add nvtracker in deepstream_test3_app have Display problem！ DeepStream SDK	3	1840	October 12, 2021
RuntimeError: get_nvds_buf_Surface: Currently we only support RGBA color Format DeepStream SDK deepstream	13	459	July 2, 2024
Test the tracker alone DeepStream SDK gstreamer	11	796	November 25, 2022
Deepstream Analytics Removefilter Isses DeepStream SDK jetson-inference , python	8	804	October 12, 2021
Error while running DS-test2 python app DeepStream SDK	5	1071	October 12, 2021
Gst-nvdewarper breaking the piepline for Gst-nvtracker DeepStream SDK gstreamer , python , deepstream	7	751	September 26, 2023
Rtsp inference and tracker DeepStream SDK ubuntu , gstreamer , python	13	250	June 29, 2024
No object when using tracker on custom nvinferserver model DeepStream SDK	3	1077	January 18, 2022
Issue regarding deepstreamer-6.4 filesink output DeepStream SDK gstreamer , computer-vision-cv , deepstream	8	328	April 30, 2024

Adding nvtracker to pipeline but nothing in OSD does not show up in the OSD

Related topics