• Hardware Platform ( GPU)
• DeepStream Version 7.0
i have a program which trying to connect both yolo and optical flow and showing output in seperate windows
python3 deepstream_object_yolov9.py /opt/nvidia/deepstream/deepstream-7.0/samples/streams/sample_720p.h264
Creating Pipeline
Creating Source
Creating H264Parser
Creating Decoder
Creating tiler
Creating nvvidconv1
Creating nv optical flow visualisation element
Creating queue
Creating queue
Creating Queue
Creating nvosd
Creating converter 2
Creating capsfilter
Creating Encoder
Creating Code Parser
Creating Container
Creating File Sink
Creating filter1
Is it Integrated GPU? : 0
Creating EGLSink
Playing file /opt/nvidia/deepstream/deepstream-7.0/samples/streams/sample_720p.h264
Adding elements to Pipeline
Traceback (most recent call last):
File “/opt/nvidia/deepstream/deepstream-7.0/sources/deepstream_python_apps/apps/deepstream-object-detection-depth/deepstream_object_yolov9.py”, line 541, in
sys.exit(main(sys.argv))
File “/opt/nvidia/deepstream/deepstream-7.0/sources/deepstream_python_apps/apps/deepstream-object-detection-depth/deepstream_object_yolov9.py”, line 439, in main
pipeline.add(nvvidconv1)
File “/usr/lib/python3/dist-packages/gi/overrides/Gst.py”, line 73, in add
raise AddError(arg)
gi.overrides.Gst.AddError: <gi.Gstnvvideoconvert object at 0x7f44e57c3d00 (Gstnvvideoconvert at 0x60fdbe48f9a0)>
iam providiing my program below
#!/usr/bin/env python3
################################################################################
SPDX-FileCopyrightText: Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
SPDX-License-Identifier: Apache-2.0
Licensed under the Apache License, Version 2.0 (the “License”);
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Apache License, Version 2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an “AS IS” BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
################################################################################
import sys
import threading
sys.path.append(‘…/’)
import os
import gi
gi.require_version(‘Gst’, ‘1.0’)
from gi.repository import GLib, Gst
from common.platform_info import PlatformInfo
from common.bus_call import bus_call
import pyds
import numpy as np
import cv2
from os import path
PGIE_CLASS_ID_PERSON = 1
PGIE_CLASS_ID_BICYCLE = 2
PGIE_CLASS_ID_CAR = 3
MUXER_BATCH_TIMEOUT_USEC = 33000
MAX_DISPLAY_LEN = 64
MUXER_OUTPUT_WIDTH = 1280
MUXER_OUTPUT_HEIGHT = 720
TILED_OUTPUT_WIDTH = 1280
TILED_OUTPUT_HEIGHT = 720
GST_CAPS_FEATURES_NVMM = “memory:NVMM”
def visualize_optical_flowvectors(flow):
“”"
Converts the flow u, v vectors into visualization by mapping them into
grey color space
:param flow: flow vectors
:return: bgr image
“”"
shape_visual = (flow.shape[0], flow.shape[1], 3)
mag, ang = cv2.cartToPolar(flow[…, 0], flow[…, 1])
hsv = np.full(shape_visual, 255, dtype=np.uint8)
hsv[…, 1] = 255
hsv[…, 0] = ang * 180 / np.pi / 2
hsv[…, 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
bgr = 255 - bgr
return bgr
def osd_src_pad_buffer_probe(pad,info,u_data):
frame_number=0
num_rects=0
gst_buffer = info.get_buffer()
if not gst_buffer:
print("Unable to get GstBuffer ")
return
# Retrieve batch metadata from the gst_buffer
# Note that pyds.gst_buffer_get_nvds_batch_meta() expects the
# C address of gst_buffer as input, which is obtained with hash(gst_buffer)
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
l_frame = batch_meta.frame_meta_list
while l_frame is not None:
try:
# Note that l_frame.data needs a cast to pyds.NvDsFrameMeta
# The casting is done by pyds.NvDsFrameMeta.cast()
# The casting also keeps ownership of the underlying memory
# in the C code, so the Python garbage collector will leave
# it alone.
frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
except StopIteration:
break
#Intiallizing object counter with 0.
obj_counter = {
PGIE_CLASS_ID_PERSON:0,
PGIE_CLASS_ID_BICYCLE:0,
PGIE_CLASS_ID_CAR:0,
}
frame_number=frame_meta.frame_num
num_rects = frame_meta.num_obj_meta
l_obj=frame_meta.obj_meta_list
l_user = frame_meta.frame_user_meta_list
save_image = False
while l_obj is not None:
try:
# Casting l_obj.data to pyds.NvDsObjectMeta
obj_meta=pyds.NvDsObjectMeta.cast(l_obj.data)
of_user_meta = pyds.NvDsUserMeta.cast(l_user.data)
except StopIteration:
break
if obj_meta.class_id in obj_counter:
obj_counter[obj_meta.class_id] += 1
left = obj_meta.detector_bbox_info.org_bbox_coords.left
top = obj_meta.detector_bbox_info.org_bbox_coords.top
width = obj_meta.detector_bbox_info.org_bbox_coords.width
height = obj_meta.detector_bbox_info.org_bbox_coords.height
print(f"bbox left: {left}, top: {top}, width: {width}, height: {height}")
try:
of_meta = pyds.NvDsOpticalFlowMeta.cast(of_user_meta.user_meta_data)
# Get Flow vectors
flow_vectors = pyds.get_optical_flow_vectors(of_meta)
# Reshape the obtained flow vectors into proper shape
flow_vectors = flow_vectors.reshape(of_meta.rows, of_meta.cols, 2)
# map the flow vectors in HSV color space for visualization
flow_visual = visualize_optical_flowvectors(flow_vectors)
got_visual = True
except StopIteration:
break
try:
l_user = l_user.next
except StopIteration:
break
if frame_number % 300 == 0:
# Getting Image data using nvbufsurface
# the input should be address of buffer and batch_id
n_frame = pyds.get_nvds_buf_surface(hash(gst_buffer), frame_meta.batch_id)
frame_copy = np.array(n_frame, copy=True, order='C')
# convert the array into cv2 default color format
frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_RGBA2BGRA)
if platform_info.is_integrated_gpu():
# If Jetson, since the buffer is mapped to CPU for retrieval, it must also be unmapped
pyds.unmap_nvds_buf_surface(hash(gst_buffer), frame_meta.batch_id) # The unmap call should be made after operations with the original array are complete.
# The original array cannot be accessed after this call.
save_image = True
obj_meta.rect_params.border_color.set(0.0, 0.0, 1.0, 0.8) #0.8 is alpha (opacity)
try:
l_obj=l_obj.next
except StopIteration:
break
# Acquiring a display meta object. The memory ownership remains in
# the C code so downstream plugins can still access it. Otherwise
# the garbage collector will claim it when this probe function exits.
display_meta=pyds.nvds_acquire_display_meta_from_pool(batch_meta)
display_meta.num_labels = 1
py_nvosd_text_params = display_meta.text_params[0]
# Setting display text to be shown on screen
# Note that the pyds module allocates a buffer for the string, and the
# memory will not be claimed by the garbage collector.
# Reading the display_text field here will return the C address of the
# allocated string. Use pyds.get_string() to get the string content.
py_nvosd_text_params.display_text = "Frame Number={} Number of Objects={} Vehicle_count={} Person_count={}".format(frame_number, num_rects, obj_counter[PGIE_CLASS_ID_CAR], obj_counter[PGIE_CLASS_ID_PERSON])
# Now set the offsets where the string should appear
py_nvosd_text_params.x_offset = 10
py_nvosd_text_params.y_offset = 12
# Font , font-color and font-size
py_nvosd_text_params.font_params.font_name = "Serif"
py_nvosd_text_params.font_params.font_size = 10
# set(red, green, blue, alpha); set to White
py_nvosd_text_params.font_params.font_color.set(1.0, 1.0, 1.0, 1.0)
# Text background color
py_nvosd_text_params.set_bg_clr = 1
# set(red, green, blue, alpha); set to Black
py_nvosd_text_params.text_bg_clr.set(0.0, 0.0, 0.0, 1.0)
# Using pyds.get_string() to get display_text as string
print(pyds.get_string(py_nvosd_text_params.display_text))
pyds.nvds_add_display_meta_to_frame(frame_meta, display_meta)
#if save_image:
# img_path = f"{folder_name}/frame_{frame_number}.jpg"
# print(f"Saving frame {frame_number} to {img_path}")
# cv2.imwrite(img_path, frame_copy)
try:
l_frame=l_frame.next
except StopIteration:
break
return Gst.PadProbeReturn.OK
def cb_newpad(decodebin, decoder_src_pad,data):
print(“In cb_newpad\n”)
caps=decoder_src_pad.get_current_caps()
gststruct=caps.get_structure(0)
gstname=gststruct.get_name()
source_bin=data
features=caps.get_features(0)
# Need to check if the pad created by the decodebin is for video and not
# audio.
print("gstname=",gstname)
if(gstname.find("video")!=-1):
# Link the decodebin pad only if decodebin has picked nvidia
# decoder plugin nvdec_*. We do this by checking if the pad caps contain
# NVMM memory features.
print("features=",features)
if features.contains("memory:NVMM"):
# Get the source bin ghost pad
bin_ghost_pad=source_bin.get_static_pad("src")
if not bin_ghost_pad.set_target(decoder_src_pad):
sys.stderr.write("Failed to link decoder src pad to source bin ghost pad\n")
else:
sys.stderr.write(" Error: Decodebin did not pick nvidia decoder plugin.\n")
def decodebin_child_added(child_proxy,Object,name,user_data):
print(“Decodebin child added:”, name, “\n”)
if(name.find(“decodebin”) != -1):
Object.connect(“child-added”,decodebin_child_added,user_data)
if "source" in name:
source_element = child_proxy.get_by_name("source")
if source_element.find_property('drop-on-latency') != None:
Object.set_property("drop-on-latency", True)
def main(args):
# Check input arguments
if len(args) < 2:
sys.stderr.write(“usage: %s \n” % args[0])
sys.exit(1)
#global folder_name
#folder_name = args[-1]
#if path.exists(folder_name):
# sys.stderr.write("The output folder %s already exists. Please remove it first.\n" % folder_name)
# sys.exit(1)
#os.mkdir(folder_name)
#print("Frames will be saved in ", folder_name)
global platform_info
platform_info = PlatformInfo()
# Standard GStreamer initialization
Gst.init(None)
# Create gstreamer elements
# Create Pipeline element that will form a connection of other elements
print("Creating Pipeline \n ")
pipeline = Gst.Pipeline()
if not pipeline:
sys.stderr.write(" Unable to create Pipeline \n")
# Source element for reading from the file
print("Creating Source \n ")
source = Gst.ElementFactory.make("filesrc", "file-source")
if not source:
sys.stderr.write(" Unable to create Source \n")
# Since the data format in the input file is elementary h264 stream,
# we need a h264parser
print("Creating H264Parser \n")
h264parser = Gst.ElementFactory.make("h264parse", "h264-parser")
if not h264parser:
sys.stderr.write(" Unable to create h264 parser \n")
# Use nvdec_h264 for hardware accelerated decode on GPU
print("Creating Decoder \n")
decoder = Gst.ElementFactory.make("nvv4l2decoder", "nvv4l2-decoder")
if not decoder:
sys.stderr.write(" Unable to create Nvv4l2 Decoder \n")
# Create nvstreammux instance to form batches from one or more sources.
streammux = Gst.ElementFactory.make("nvstreammux", "Stream-muxer")
if not streammux:
sys.stderr.write(" Unable to create NvStreamMux \n")
queue1=Gst.ElementFactory.make("queue","queue1")
queue2=Gst.ElementFactory.make("queue","queue2")
queue3=Gst.ElementFactory.make("queue","queue3")
queue4=Gst.ElementFactory.make("queue","queue4")
queue5=Gst.ElementFactory.make("queue","queue5")
pipeline.add(queue1)
pipeline.add(queue2)
pipeline.add(queue3)
pipeline.add(queue4)
pipeline.add(queue5)
# Use nvinfer to run inferencing on decoder's output,
# behaviour of inferencing is set through config file
pgie = Gst.ElementFactory.make("nvinfer", "primary-inference")
if not pgie:
sys.stderr.write(" Unable to create pgie \n")
# Use convertor to convert from NV12 to RGBA as required by nvosd
nvvidconv = Gst.ElementFactory.make("nvvideoconvert", "convertor")
if not nvvidconv:
sys.stderr.write(" Unable to create nvvidconv \n")
print("Creating tiler \n ")
tiler=Gst.ElementFactory.make("nvmultistreamtiler", "nvtiler")
if not tiler:
sys.stderr.write(" Unable to create tiler \n")
print("Creating nvvidconv1 \n ")
nvvidconv1 = Gst.ElementFactory.make("nvvideoconvert", "convertor")
if not nvvidconv1:
sys.stderr.write(" Unable to create nvvidconv \n")
nvof = Gst.ElementFactory.make("nvof", "nvopticalflow")
if not nvof:
sys.stderr.write("Unable to create optical flow \n")
print("Creating nv optical flow visualisation element \n")
nvofvisual = Gst.ElementFactory.make("nvofvisual", "nvopticalflowvisual")
if not nvofvisual:
sys.stderr.write("Unable to create flow visualisation element")
print("Creating queue \n ")
of_queue = Gst.ElementFactory.make("queue", "q_after_of")
if not of_queue:
sys.stderr.write("Unable to create queue \n")
print("Creating queue \n")
ofvisual_queue = Gst.ElementFactory.make("queue", "q_after_ofvisual")
if not ofvisual_queue:
sys.stderr.write("Unable to create queue \n")
print("Creating Queue \n")
queue = Gst.ElementFactory.make("queue", "queue")
if not queue:
sys.stderr.write(" Unable to create queue \n")
print("Creating nvosd \n ")
nvosd1 = Gst.ElementFactory.make("nvdsosd", "onscreendisplay")
if not nvosd1:
sys.stderr.write(" Unable to create nvosd \n")
print("Creating converter 2\n")
nvvidconv2 = Gst.ElementFactory.make("nvvideoconvert", "convertor2")
if not nvvidconv2:
sys.stderr.write(" Unable to create nvvidconv2 \n")
print("Creating capsfilter \n")
capsfilter = Gst.ElementFactory.make("capsfilter", "capsfilter")
if not capsfilter:
sys.stderr.write(" Unable to create capsfilter \n")
caps = Gst.Caps.from_string("video/x-raw, format=I420")
capsfilter.set_property("caps", caps)
print("Creating Encoder \n")
encoder = Gst.ElementFactory.make("avenc_mpeg4", "encoder")
if not encoder:
sys.stderr.write(" Unable to create encoder \n")
encoder.set_property("bitrate", 2000000)
print("Creating Code Parser \n")
codeparser = Gst.ElementFactory.make("mpeg4videoparse", "mpeg4-parser")
if not codeparser:
sys.stderr.write(" Unable to create code parser \n")
print("Creating Container \n")
container = Gst.ElementFactory.make("qtmux", "qtmux")
if not container:
sys.stderr.write(" Unable to create code parser \n")
print("Creating File Sink \n")
sink1 = Gst.ElementFactory.make("filesink", "filesink")
if not sink1:
sys.stderr.write(" Unable to create file sink \n")
sink1.set_property("location", "./out.mp4")
sink1.set_property("sync", 0)
print("Creating filter1 \n ")
caps1 = Gst.Caps.from_string("video/x-raw(memory:NVMM), format=RGBA")
filter1 = Gst.ElementFactory.make("capsfilter", "filter1")
if not filter1:
sys.stderr.write(" Unable to get the caps filter1 \n")
filter1.set_property("caps", caps1)
# Create OSD to draw on the converted RGBA buffer
nvosd = Gst.ElementFactory.make("nvdsosd", "onscreendisplay")
if not nvosd:
sys.stderr.write(" Unable to create nvosd \n")
# Finally render the osd output
if platform_info.is_integrated_gpu():
print("Creating nv3dsink \n")
sink = Gst.ElementFactory.make("nv3dsink", "nv3d-sink")
if not sink:
sys.stderr.write(" Unable to create nv3dsink \n")
else:
if platform_info.is_platform_aarch64():
print("Creating nv3dsink \n")
sink = Gst.ElementFactory.make("nv3dsink", "nv3d-sink")
else:
print("Creating EGLSink \n")
sink = Gst.ElementFactory.make("nveglglessink", "nvvideo-renderer")
sink.set_property('sync', True)
if not sink:
sys.stderr.write(" Unable to create egl sink \n")
print("Playing file %s " %args[1])
source.set_property('location', args[1])
if os.environ.get('USE_NEW_NVSTREAMMUX') != 'yes': # Only set these properties if not using new gst-nvstreammux
streammux.set_property('width', 1920)
streammux.set_property('height', 1080)
streammux.set_property('batched-push-timeout', MUXER_BATCH_TIMEOUT_USEC)
streammux.set_property('batch-size', 1)
pgie.set_property('config-file-path', "config_pgie_yolov9_det.txt")
if not platform_info.is_integrated_gpu():
# Use CUDA unified memory in the pipeline so frames
# can be easily accessed on CPU in Python.
mem_type = int(pyds.NVBUF_MEM_CUDA_UNIFIED)
streammux.set_property("nvbuf-memory-type", mem_type)
nvvidconv.set_property("nvbuf-memory-type", mem_type)
nvvidconv1.set_property("nvbuf-memory-type", mem_type)
nvvidconv2.set_property("nvbuf-memory-type", mem_type)
print("Adding elements to Pipeline \n")
pipeline.add(source)
pipeline.add(h264parser)
pipeline.add(decoder)
pipeline.add(streammux)
pipeline.add(pgie)
pipeline.add(nvvidconv)
pipeline.add(filter1)
pipeline.add(nvosd)
pipeline.add(sink)
pipeline.add(nvof)
pipeline.add(of_queue)
pipeline.add(nvofvisual)
pipeline.add(ofvisual_queue)
pipeline.add(tiler)
pipeline.add(nvvidconv1)
pipeline.add(nvosd1)
pipeline.add(nvvidconv2)
pipeline.add(capsfilter)
pipeline.add(encoder)
pipeline.add(codeparser)
pipeline.add(container)
pipeline.add(sink1)
# we link the elements together
# file-source -> h264-parser -> nvh264-decoder ->
# nvinfer -> nvvidconv -> nvosd -> video-renderer
print("Linking elements in the Pipeline \n")
source.link(h264parser)
h264parser.link(decoder)
sinkpad = streammux.request_pad_simple("sink_0")
if not sinkpad:
sys.stderr.write(" Unable to get the sink pad of streammux \n")
srcpad = decoder.get_static_pad("src")
if not srcpad:
sys.stderr.write(" Unable to get source pad of decoder \n")
srcpad.link(sinkpad)
streammux.link(pgie)
pgie.link(nvvidconv)
nvvidconv.link(filter1)
filter1.link(nvosd)
nvosd.link(sink)
print(“Linking elements in the Pipeline \n”)
streammux.link(queue1)
queue1.link(nvof)
nvof.link(of_queue)
of_queue.link(nvofvisual)
nvofvisual.link(ofvisual_queue)
ofvisual_queue.link(tiler)
tiler.link(queue3)
queue3.link(nvvidconv)
nvvidconv.link(queue4)
queue4.link(nvosd)
nvosd.link(queue5)
queue5.link(nvvidconv2)
nvvidconv2.link(capsfilter)
capsfilter.link(encoder)
encoder.link(codeparser)
codeparser.link(container)
container.link(sink)
streammux.link(queue1)
queue1.link(pgie)
queue1.link(nvof)
nvof.link(of_queue)
of_queue.link(nvofvisual)
nvofvisual.link(ofvisual_queue)
ofvisual_queue.link(tiler)
tiler.link(queue3)
pgie.link(nvvidconv)
queue3.link(nvvidconv1)
nvvidconv1.link(queue4)
nvvidconv.link(filter1)
filter1.link(nvosd)
queue4.link(nvosd1)
nvosd1.link(queue5)
queue5.link(nvvidconv2)
nvvidconv2.link(capsfilter)
capsfilter.link(encoder)
encoder.link(codeparser)
codeparser.link(container)
nvosd.link(sink)
container.link(sink1)
# create an event loop and feed gstreamer bus mesages to it
loop = GLib.MainLoop()
bus = pipeline.get_bus()
bus.add_signal_watch()
bus.connect ("message", bus_call, loop)
# Lets add probe to get informed of the meta data generated, we add probe to
# the sink pad of the osd element, since by that time, the buffer would have
# had got all the metadata.
osdsrcpad = nvosd.get_static_pad("src")
if not osdsrcpad:
sys.stderr.write(" Unable to get src pad of nvosd \n")
osdsrcpad.add_probe(Gst.PadProbeType.BUFFER, osd_src_pad_buffer_probe, 0)
# start play back and listen to events
print("Starting pipeline \n")
pipeline.set_state(Gst.State.PLAYING)
try:
loop.run()
except:
pass
# cleanup
pipeline.set_state(Gst.State.NULL)
if name == ‘main’:
sys.exit(main(sys.argv))