Yolo and Optical Flow integration

• Hardware Platform ( GPU)
• DeepStream Version 7.0

i have a program which trying to connect both yolo and optical flow and showing output in seperate windows
python3 deepstream_object_yolov9.py /opt/nvidia/deepstream/deepstream-7.0/samples/streams/sample_720p.h264
Creating Pipeline

Creating Source

Creating H264Parser

Creating Decoder

Creating tiler

Creating nvvidconv1

Creating nv optical flow visualisation element

Creating queue

Creating queue

Creating Queue

Creating nvosd

Creating converter 2

Creating capsfilter

Creating Encoder

Creating Code Parser

Creating Container

Creating File Sink

Creating filter1

Is it Integrated GPU? : 0
Creating EGLSink

Playing file /opt/nvidia/deepstream/deepstream-7.0/samples/streams/sample_720p.h264
Adding elements to Pipeline

Traceback (most recent call last):
File “/opt/nvidia/deepstream/deepstream-7.0/sources/deepstream_python_apps/apps/deepstream-object-detection-depth/deepstream_object_yolov9.py”, line 541, in
sys.exit(main(sys.argv))
File “/opt/nvidia/deepstream/deepstream-7.0/sources/deepstream_python_apps/apps/deepstream-object-detection-depth/deepstream_object_yolov9.py”, line 439, in main
pipeline.add(nvvidconv1)
File “/usr/lib/python3/dist-packages/gi/overrides/Gst.py”, line 73, in add
raise AddError(arg)
gi.overrides.Gst.AddError: <gi.Gstnvvideoconvert object at 0x7f44e57c3d00 (Gstnvvideoconvert at 0x60fdbe48f9a0)>

iam providiing my program below
#!/usr/bin/env python3

################################################################################

SPDX-FileCopyrightText: Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.

SPDX-License-Identifier: Apache-2.0

Licensed under the Apache License, Version 2.0 (the “License”);

you may not use this file except in compliance with the License.

You may obtain a copy of the License at

Apache License, Version 2.0

Unless required by applicable law or agreed to in writing, software

distributed under the License is distributed on an “AS IS” BASIS,

WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

See the License for the specific language governing permissions and

limitations under the License.

################################################################################

import sys
import threading
sys.path.append(‘…/’)
import os
import gi
gi.require_version(‘Gst’, ‘1.0’)
from gi.repository import GLib, Gst
from common.platform_info import PlatformInfo
from common.bus_call import bus_call

import pyds
import numpy as np
import cv2
from os import path

PGIE_CLASS_ID_PERSON = 1
PGIE_CLASS_ID_BICYCLE = 2
PGIE_CLASS_ID_CAR = 3
MUXER_BATCH_TIMEOUT_USEC = 33000
MAX_DISPLAY_LEN = 64
MUXER_OUTPUT_WIDTH = 1280
MUXER_OUTPUT_HEIGHT = 720
TILED_OUTPUT_WIDTH = 1280
TILED_OUTPUT_HEIGHT = 720
GST_CAPS_FEATURES_NVMM = “memory:NVMM”

def visualize_optical_flowvectors(flow):
“”"
Converts the flow u, v vectors into visualization by mapping them into
grey color space
:param flow: flow vectors
:return: bgr image
“”"
shape_visual = (flow.shape[0], flow.shape[1], 3)
mag, ang = cv2.cartToPolar(flow[…, 0], flow[…, 1])
hsv = np.full(shape_visual, 255, dtype=np.uint8)
hsv[…, 1] = 255
hsv[…, 0] = ang * 180 / np.pi / 2
hsv[…, 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
bgr = 255 - bgr
return bgr

def osd_src_pad_buffer_probe(pad,info,u_data):
frame_number=0
num_rects=0

gst_buffer = info.get_buffer()
if not gst_buffer:
    print("Unable to get GstBuffer ")
    return

# Retrieve batch metadata from the gst_buffer
# Note that pyds.gst_buffer_get_nvds_batch_meta() expects the
# C address of gst_buffer as input, which is obtained with hash(gst_buffer)
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
l_frame = batch_meta.frame_meta_list
while l_frame is not None:
    try:
        # Note that l_frame.data needs a cast to pyds.NvDsFrameMeta
        # The casting is done by pyds.NvDsFrameMeta.cast()
        # The casting also keeps ownership of the underlying memory
        # in the C code, so the Python garbage collector will leave
        # it alone.
        frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
    except StopIteration:
        break

    #Intiallizing object counter with 0.
    obj_counter = {
        PGIE_CLASS_ID_PERSON:0,
        PGIE_CLASS_ID_BICYCLE:0,
        PGIE_CLASS_ID_CAR:0,
    }
    frame_number=frame_meta.frame_num
    num_rects = frame_meta.num_obj_meta
    l_obj=frame_meta.obj_meta_list
    l_user = frame_meta.frame_user_meta_list
    save_image = False
    while l_obj is not None:
        try:
            # Casting l_obj.data to pyds.NvDsObjectMeta
            obj_meta=pyds.NvDsObjectMeta.cast(l_obj.data)
            of_user_meta = pyds.NvDsUserMeta.cast(l_user.data)
        except StopIteration:
            break
        if obj_meta.class_id in obj_counter:
            obj_counter[obj_meta.class_id] += 1
        left = obj_meta.detector_bbox_info.org_bbox_coords.left
        top = obj_meta.detector_bbox_info.org_bbox_coords.top
        width = obj_meta.detector_bbox_info.org_bbox_coords.width
        height = obj_meta.detector_bbox_info.org_bbox_coords.height
        print(f"bbox left: {left}, top: {top}, width: {width}, height: {height}")
        
        try:
            of_meta = pyds.NvDsOpticalFlowMeta.cast(of_user_meta.user_meta_data)
            # Get Flow vectors
            flow_vectors = pyds.get_optical_flow_vectors(of_meta)
        # Reshape the obtained flow vectors into proper shape
            flow_vectors = flow_vectors.reshape(of_meta.rows, of_meta.cols, 2)
            # map the flow vectors in HSV color space for visualization
            flow_visual = visualize_optical_flowvectors(flow_vectors)
            got_visual = True
        except StopIteration:
            break
        try:
            l_user = l_user.next
        except StopIteration:
            break
        

        if frame_number % 300 == 0:
            # Getting Image data using nvbufsurface
            # the input should be address of buffer and batch_id
            n_frame = pyds.get_nvds_buf_surface(hash(gst_buffer), frame_meta.batch_id)
            frame_copy = np.array(n_frame, copy=True, order='C')
            # convert the array into cv2 default color format
            frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_RGBA2BGRA)
            if platform_info.is_integrated_gpu():
                # If Jetson, since the buffer is mapped to CPU for retrieval, it must also be unmapped 
                pyds.unmap_nvds_buf_surface(hash(gst_buffer), frame_meta.batch_id) # The unmap call should be made after operations with the original array are complete.
                                                                                        #  The original array cannot be accessed after this call.
            save_image = True

        obj_meta.rect_params.border_color.set(0.0, 0.0, 1.0, 0.8) #0.8 is alpha (opacity)
        try: 
            l_obj=l_obj.next
        except StopIteration:
            break

    # Acquiring a display meta object. The memory ownership remains in
    # the C code so downstream plugins can still access it. Otherwise
    # the garbage collector will claim it when this probe function exits.
    display_meta=pyds.nvds_acquire_display_meta_from_pool(batch_meta)
    display_meta.num_labels = 1
    py_nvosd_text_params = display_meta.text_params[0]
    # Setting display text to be shown on screen
    # Note that the pyds module allocates a buffer for the string, and the
    # memory will not be claimed by the garbage collector.
    # Reading the display_text field here will return the C address of the
    # allocated string. Use pyds.get_string() to get the string content.
    py_nvosd_text_params.display_text = "Frame Number={} Number of Objects={} Vehicle_count={} Person_count={}".format(frame_number, num_rects, obj_counter[PGIE_CLASS_ID_CAR], obj_counter[PGIE_CLASS_ID_PERSON])

    # Now set the offsets where the string should appear
    py_nvosd_text_params.x_offset = 10
    py_nvosd_text_params.y_offset = 12

    # Font , font-color and font-size
    py_nvosd_text_params.font_params.font_name = "Serif"
    py_nvosd_text_params.font_params.font_size = 10
    # set(red, green, blue, alpha); set to White
    py_nvosd_text_params.font_params.font_color.set(1.0, 1.0, 1.0, 1.0)

    # Text background color
    py_nvosd_text_params.set_bg_clr = 1
    # set(red, green, blue, alpha); set to Black
    py_nvosd_text_params.text_bg_clr.set(0.0, 0.0, 0.0, 1.0)
    # Using pyds.get_string() to get display_text as string
    print(pyds.get_string(py_nvosd_text_params.display_text))
    pyds.nvds_add_display_meta_to_frame(frame_meta, display_meta)
    
    #if save_image:
    #    img_path = f"{folder_name}/frame_{frame_number}.jpg"
    #    print(f"Saving frame {frame_number} to {img_path}")
    #    cv2.imwrite(img_path, frame_copy)
    try:
        l_frame=l_frame.next
    except StopIteration:
        break
return Gst.PadProbeReturn.OK

def cb_newpad(decodebin, decoder_src_pad,data):
print(“In cb_newpad\n”)
caps=decoder_src_pad.get_current_caps()
gststruct=caps.get_structure(0)
gstname=gststruct.get_name()
source_bin=data
features=caps.get_features(0)

# Need to check if the pad created by the decodebin is for video and not
# audio.
print("gstname=",gstname)
if(gstname.find("video")!=-1):
    # Link the decodebin pad only if decodebin has picked nvidia
    # decoder plugin nvdec_*. We do this by checking if the pad caps contain
    # NVMM memory features.
    print("features=",features)
    if features.contains("memory:NVMM"):
        # Get the source bin ghost pad
        bin_ghost_pad=source_bin.get_static_pad("src")
        if not bin_ghost_pad.set_target(decoder_src_pad):
            sys.stderr.write("Failed to link decoder src pad to source bin ghost pad\n")
    else:
        sys.stderr.write(" Error: Decodebin did not pick nvidia decoder plugin.\n")

def decodebin_child_added(child_proxy,Object,name,user_data):
print(“Decodebin child added:”, name, “\n”)
if(name.find(“decodebin”) != -1):
Object.connect(“child-added”,decodebin_child_added,user_data)

if "source" in name:
    source_element = child_proxy.get_by_name("source")
    if source_element.find_property('drop-on-latency') != None:
        Object.set_property("drop-on-latency", True)

def main(args):
# Check input arguments
if len(args) < 2:
sys.stderr.write(“usage: %s \n” % args[0])
sys.exit(1)

#global folder_name
#folder_name = args[-1]
#if path.exists(folder_name):
#    sys.stderr.write("The output folder %s already exists. Please remove it first.\n" % folder_name)
 #   sys.exit(1)

#os.mkdir(folder_name)
#print("Frames will be saved in ", folder_name)

global platform_info
platform_info = PlatformInfo()
# Standard GStreamer initialization
Gst.init(None)

# Create gstreamer elements
# Create Pipeline element that will form a connection of other elements
print("Creating Pipeline \n ")
pipeline = Gst.Pipeline()

if not pipeline:
    sys.stderr.write(" Unable to create Pipeline \n")

# Source element for reading from the file
print("Creating Source \n ")
source = Gst.ElementFactory.make("filesrc", "file-source")
if not source:
    sys.stderr.write(" Unable to create Source \n")

# Since the data format in the input file is elementary h264 stream,
# we need a h264parser
print("Creating H264Parser \n")
h264parser = Gst.ElementFactory.make("h264parse", "h264-parser")
if not h264parser:
    sys.stderr.write(" Unable to create h264 parser \n")

# Use nvdec_h264 for hardware accelerated decode on GPU
print("Creating Decoder \n")
decoder = Gst.ElementFactory.make("nvv4l2decoder", "nvv4l2-decoder")
if not decoder:
    sys.stderr.write(" Unable to create Nvv4l2 Decoder \n")

# Create nvstreammux instance to form batches from one or more sources.
streammux = Gst.ElementFactory.make("nvstreammux", "Stream-muxer")
if not streammux:
    sys.stderr.write(" Unable to create NvStreamMux \n")
    
queue1=Gst.ElementFactory.make("queue","queue1")
queue2=Gst.ElementFactory.make("queue","queue2")
queue3=Gst.ElementFactory.make("queue","queue3")
queue4=Gst.ElementFactory.make("queue","queue4")
queue5=Gst.ElementFactory.make("queue","queue5")
pipeline.add(queue1)
pipeline.add(queue2)
pipeline.add(queue3)
pipeline.add(queue4)
pipeline.add(queue5)

# Use nvinfer to run inferencing on decoder's output,
# behaviour of inferencing is set through config file
pgie = Gst.ElementFactory.make("nvinfer", "primary-inference")
if not pgie:
    sys.stderr.write(" Unable to create pgie \n")

# Use convertor to convert from NV12 to RGBA as required by nvosd
nvvidconv = Gst.ElementFactory.make("nvvideoconvert", "convertor")
if not nvvidconv:
    sys.stderr.write(" Unable to create nvvidconv \n")
    
    
    

print("Creating tiler \n ")
tiler=Gst.ElementFactory.make("nvmultistreamtiler", "nvtiler")
if not tiler:
    sys.stderr.write(" Unable to create tiler \n")
print("Creating nvvidconv1 \n ")
nvvidconv1 = Gst.ElementFactory.make("nvvideoconvert", "convertor")
if not nvvidconv1:
    sys.stderr.write(" Unable to create nvvidconv \n")

nvof = Gst.ElementFactory.make("nvof", "nvopticalflow")
if not nvof:
    sys.stderr.write("Unable to create optical flow \n")
print("Creating nv optical flow visualisation element \n")
nvofvisual = Gst.ElementFactory.make("nvofvisual", "nvopticalflowvisual")
if not nvofvisual:
    sys.stderr.write("Unable to create flow visualisation element")
print("Creating queue \n ")
of_queue = Gst.ElementFactory.make("queue", "q_after_of")
if not of_queue:
    sys.stderr.write("Unable to create queue \n")
print("Creating queue \n")
ofvisual_queue = Gst.ElementFactory.make("queue", "q_after_ofvisual")
if not ofvisual_queue:
    sys.stderr.write("Unable to create queue \n")

print("Creating Queue \n")
queue = Gst.ElementFactory.make("queue", "queue")
if not queue:
    sys.stderr.write(" Unable to create queue \n")
print("Creating nvosd \n ")
nvosd1 = Gst.ElementFactory.make("nvdsosd", "onscreendisplay")
if not nvosd1:
    sys.stderr.write(" Unable to create nvosd \n")

print("Creating converter 2\n")
nvvidconv2 = Gst.ElementFactory.make("nvvideoconvert", "convertor2")
if not nvvidconv2:
    sys.stderr.write(" Unable to create nvvidconv2 \n")

print("Creating capsfilter \n")
capsfilter = Gst.ElementFactory.make("capsfilter", "capsfilter")
if not capsfilter:
    sys.stderr.write(" Unable to create capsfilter \n")
caps = Gst.Caps.from_string("video/x-raw, format=I420")
capsfilter.set_property("caps", caps)
print("Creating Encoder \n")
encoder = Gst.ElementFactory.make("avenc_mpeg4", "encoder")
if not encoder:
    sys.stderr.write(" Unable to create encoder \n")
encoder.set_property("bitrate", 2000000)
print("Creating Code Parser \n")
codeparser = Gst.ElementFactory.make("mpeg4videoparse", "mpeg4-parser")
if not codeparser:
    sys.stderr.write(" Unable to create code parser \n")
print("Creating Container \n")
container = Gst.ElementFactory.make("qtmux", "qtmux")
if not container:
    sys.stderr.write(" Unable to create code parser \n")
print("Creating File Sink \n")
sink1 = Gst.ElementFactory.make("filesink", "filesink")
if not sink1:
    sys.stderr.write(" Unable to create file sink \n")
sink1.set_property("location", "./out.mp4")
sink1.set_property("sync", 0)








print("Creating filter1 \n ")
caps1 = Gst.Caps.from_string("video/x-raw(memory:NVMM), format=RGBA")
filter1 = Gst.ElementFactory.make("capsfilter", "filter1")
if not filter1:
    sys.stderr.write(" Unable to get the caps filter1 \n")
filter1.set_property("caps", caps1)

# Create OSD to draw on the converted RGBA buffer
nvosd = Gst.ElementFactory.make("nvdsosd", "onscreendisplay")
if not nvosd:
    sys.stderr.write(" Unable to create nvosd \n")

# Finally render the osd output
if platform_info.is_integrated_gpu():
    print("Creating nv3dsink \n")
    sink = Gst.ElementFactory.make("nv3dsink", "nv3d-sink")
    if not sink:
        sys.stderr.write(" Unable to create nv3dsink \n")
else:
    if platform_info.is_platform_aarch64():
        print("Creating nv3dsink \n")
        sink = Gst.ElementFactory.make("nv3dsink", "nv3d-sink")
    else:
        print("Creating EGLSink \n")
        sink = Gst.ElementFactory.make("nveglglessink", "nvvideo-renderer")
        sink.set_property('sync', True)
    if not sink:
        sys.stderr.write(" Unable to create egl sink \n")

print("Playing file %s " %args[1])
source.set_property('location', args[1])
if os.environ.get('USE_NEW_NVSTREAMMUX') != 'yes': # Only set these properties if not using new gst-nvstreammux
    streammux.set_property('width', 1920)
    streammux.set_property('height', 1080)
    streammux.set_property('batched-push-timeout', MUXER_BATCH_TIMEOUT_USEC)

streammux.set_property('batch-size', 1)
pgie.set_property('config-file-path', "config_pgie_yolov9_det.txt")

if not platform_info.is_integrated_gpu():
    # Use CUDA unified memory in the pipeline so frames
    # can be easily accessed on CPU in Python.
    mem_type = int(pyds.NVBUF_MEM_CUDA_UNIFIED)
    streammux.set_property("nvbuf-memory-type", mem_type)
    nvvidconv.set_property("nvbuf-memory-type", mem_type)
    nvvidconv1.set_property("nvbuf-memory-type", mem_type)
    nvvidconv2.set_property("nvbuf-memory-type", mem_type)

print("Adding elements to Pipeline \n")
pipeline.add(source)
pipeline.add(h264parser)
pipeline.add(decoder)
pipeline.add(streammux)
pipeline.add(pgie)
pipeline.add(nvvidconv)
pipeline.add(filter1)
pipeline.add(nvosd)
pipeline.add(sink)

pipeline.add(nvof)
pipeline.add(of_queue)
pipeline.add(nvofvisual)
pipeline.add(ofvisual_queue)
pipeline.add(tiler)
pipeline.add(nvvidconv1)
pipeline.add(nvosd1)
pipeline.add(nvvidconv2)
pipeline.add(capsfilter)
pipeline.add(encoder)
pipeline.add(codeparser)
pipeline.add(container)
pipeline.add(sink1)

# we link the elements together
# file-source -> h264-parser -> nvh264-decoder ->
# nvinfer -> nvvidconv -> nvosd -> video-renderer
print("Linking elements in the Pipeline \n")
source.link(h264parser)
h264parser.link(decoder)

sinkpad = streammux.request_pad_simple("sink_0")
if not sinkpad:
    sys.stderr.write(" Unable to get the sink pad of streammux \n")
srcpad = decoder.get_static_pad("src")
if not srcpad:
    sys.stderr.write(" Unable to get source pad of decoder \n")
srcpad.link(sinkpad)

streammux.link(pgie)

pgie.link(nvvidconv)

nvvidconv.link(filter1)

filter1.link(nvosd)

nvosd.link(sink)

print(“Linking elements in the Pipeline \n”)

streammux.link(queue1)

queue1.link(nvof)

nvof.link(of_queue)

of_queue.link(nvofvisual)

nvofvisual.link(ofvisual_queue)

ofvisual_queue.link(tiler)

tiler.link(queue3)

queue3.link(nvvidconv)

nvvidconv.link(queue4)

queue4.link(nvosd)

nvosd.link(queue5)

queue5.link(nvvidconv2)

nvvidconv2.link(capsfilter)

capsfilter.link(encoder)

encoder.link(codeparser)

codeparser.link(container)

container.link(sink)

streammux.link(queue1)
queue1.link(pgie)
queue1.link(nvof)
nvof.link(of_queue)
of_queue.link(nvofvisual)
nvofvisual.link(ofvisual_queue)
ofvisual_queue.link(tiler)
tiler.link(queue3)
pgie.link(nvvidconv)
queue3.link(nvvidconv1)
nvvidconv1.link(queue4)
nvvidconv.link(filter1)
filter1.link(nvosd)
queue4.link(nvosd1)
nvosd1.link(queue5)
queue5.link(nvvidconv2)
nvvidconv2.link(capsfilter)
capsfilter.link(encoder)
encoder.link(codeparser)
codeparser.link(container)
nvosd.link(sink)
container.link(sink1)


  

# create an event loop and feed gstreamer bus mesages to it
loop = GLib.MainLoop()
bus = pipeline.get_bus()
bus.add_signal_watch()
bus.connect ("message", bus_call, loop)

# Lets add probe to get informed of the meta data generated, we add probe to
# the sink pad of the osd element, since by that time, the buffer would have
# had got all the metadata.
osdsrcpad = nvosd.get_static_pad("src")
if not osdsrcpad:
    sys.stderr.write(" Unable to get src pad of nvosd \n")

osdsrcpad.add_probe(Gst.PadProbeType.BUFFER, osd_src_pad_buffer_probe, 0)

# start play back and listen to events
print("Starting pipeline \n")
pipeline.set_state(Gst.State.PLAYING)

try:
    loop.run()
except:
    pass
# cleanup
pipeline.set_state(Gst.State.NULL)

if name == ‘main’:
sys.exit(main(sys.argv))

this is a Gstreamer usage issue. In the code, nvvidconv1 and nvvidconv are the same kind of plugin. they should use different plugin name. please modify the code to
nvvidconv = Gst.ElementFactory.make(“nvvideoconvert”, “convertor”)

nvvidconv1 = Gst.ElementFactory.make(“nvvideoconvert”, “convertor1”)

Thank you for the solution
now similar issue faced in nvosd that also solved like above

but now iam facing a new error
Traceback (most recent call last):
File “/opt/nvidia/deepstream/deepstream-7.0/sources/deepstream_python_apps/apps/deepstream-object-detection-depth/deepstream_object_yolov9.py”, line 100, in osd_src_pad_buffer_probe
of_user_meta = pyds.NvDsUserMeta.cast(l_user.data)
AttributeError: ‘NoneType’ object has no attribute ‘data’

please refer to this code for how to get user_meta.