• Hardware Platform (Jetson / GPU)
GPUs: RTX 2070S, A100 and V100
• DeepStream Version
5.1 (nvcr.io/nvidia/deepstream:5.1-21.02-devel container)
• TensorRT Version
7.2.2.3
• NVIDIA GPU Driver Version (valid for GPU only)
RTX2070S: 470.63.01
A100: 470.57.02
V100: 455.45.01
• Issue Type( questions, new requirements, bugs)
bug
• How to reproduce the issue ? (This is for bugs. Including which sample app is using, the configuration files content, the command line used and other details for reproducing)
I’m trying to restart the pipeline by setting it to NULL and then to PLAYING when specific problems happen (like when the rtsp server is temporarily unavailable) as one of the implemented ways to keep the service always alive. After the first try to set the pipeline to NULL and then to PLAYING the bounding boxes that should be draw by nvdsosd won’t appear anymore, and after more few tries the app crashes with “Segmentation fault (core dumped)” without throwing errors or warnings just after going to PLAYING state.
This problem only happens when nvinfer plugin is in the pipeline.
I’m using the code above to reproduce the issue on the different machines (using fakesink instead of nveglglessink on A100 and V100). It uses a callback function that restarts the pipeline every 30 seconds.
import sys, json
import gi, pyds, configparser
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst, GLib
from ds_utils.is_aarch_64 import is_aarch64
def bus_call(bus, message, loop):
t = message.type
if t == Gst.MessageType.EOS:
sys.stdout.write("End-of-stream\n")
loop.quit()
elif t == Gst.MessageType.ERROR:
err, debug = message.parse_error()
sys.stderr.write("Error: %s: %s\n" % (err, debug))
loop.quit()
return True
def cb_newpad(decodebin, decoder_src_pad, sinkpad):
caps=decoder_src_pad.get_current_caps()
gststruct=caps.get_structure(0)
gstname=gststruct.get_name()
features=caps.get_features(0)
if(gstname.find("video")!=-1):
decoder_src_pad.link(sinkpad)
def play_if_null(pipeline):
ret , state, pending = pipeline.get_state(0)
if ret == Gst.StateChangeReturn.SUCCESS and state == Gst.State.NULL:
pipeline.set_state(Gst.State.PLAYING)
return False
return True
def restart_pipeline(pipeline):
state_ret = pipeline.set_state(Gst.State.NULL)
GLib.timeout_add(1000, play_if_null, pipeline)
return True
def main(args):
if len(args) != 2:
sys.stderr.write("usage: %s <media file or uri>\n" % args[0])
sys.exit(1)
GObject.threads_init()
Gst.init(None)
pipeline = Gst.Pipeline()
uridecodebin = Gst.ElementFactory.make("uridecodebin", "sourcebin"); assert uridecodebin
if not uridecodebin:
sys.stderr.write("'uridecodebin' gstreamer plugin missing\n")
sys.exit(1)
streammux = Gst.ElementFactory.make("nvstreammux", "mux")
streammux.set_property("width", 640)
streammux.set_property("height", 480)
streammux.set_property("batched-push-timeout", 40000)
streammux.set_property("batch-size", 1)
streammux.set_property("live-source", 1)
detector = Gst.ElementFactory.make("nvinfer", "primary-inference"); assert detector
tracker = Gst.ElementFactory.make("nvtracker", "tracker"); assert tracker
nvdsanalytics = Gst.ElementFactory.make("nvdsanalytics", "nvdsanalytics"); assert nvdsanalytics
# Setting elements properties
detector.set_property('config-file-path', "/app/config/nvinfer_detector_config.txt")
pgie_batch_size=detector.get_property("batch-size")
if(pgie_batch_size != 1):
print(f"Overriding infer-config batch-size {pgie_batch_size} with max batch size 1")
detector.set_property("batch-size",1)
tracker_config = configparser.ConfigParser()
tracker_config.read("/app/config/nvtracker_tracker_config.txt")
tracker_config.sections()
for key in tracker_config['tracker']:
if key == 'tracker-width' :
tracker_width = tracker_config.getint('tracker', key)
tracker.set_property('tracker-width', tracker_width)
if key == 'tracker-height' :
tracker_height = tracker_config.getint('tracker', key)
tracker.set_property('tracker-height', tracker_height)
if key == 'gpu-id' :
tracker_gpu_id = tracker_config.getint('tracker', key)
tracker.set_property('gpu_id', tracker_gpu_id)
if key == 'll-lib-file' :
tracker_ll_lib_file = tracker_config.get('tracker', key)
tracker.set_property('ll-lib-file', tracker_ll_lib_file)
if key == 'll-config-file' :
tracker_ll_config_file = tracker_config.get('tracker', key)
tracker.set_property('ll-config-file', tracker_ll_config_file)
if key == 'enable-batch-process' :
tracker_enable_batch_process = tracker_config.getint('tracker', key)
tracker.set_property('enable_batch_process', tracker_enable_batch_process)
nvdsanalytics.set_property('config-file', "/app/config/config_nvdsanalytics.txt")
nvdsanalytics.set_property('enable', 1)
tiler = Gst.ElementFactory.make("nvmultistreamtiler", "tiler"); assert tiler
tiler.set_property("width", 640)
tiler.set_property("height", 480)
tiler.set_property("columns",1)
tiler.set_property("rows",1)
osd_convertor = Gst.ElementFactory.make("nvvideoconvert", "osd_convertor"); assert osd_convertor
nvosd = Gst.ElementFactory.make("nvdsosd", "onscreendisplay"); assert nvosd
nvosd.set_property('process-mode', 1)
nvosd.set_property('display-text', 1)
if is_aarch64():
transform = Gst.ElementFactory.make("nvegltransform", "nvegl-transform"); assert transform
videosink = Gst.ElementFactory.make("nveglglessink", "videosink"); assert videosink
videosink.set_property("sync", 0)
pipeline.add(uridecodebin)
pipeline.add(streammux)
pipeline.add(detector)
pipeline.add(tracker)
pipeline.add(nvdsanalytics)
pipeline.add(tiler)
pipeline.add(videosink)
pipeline.add(osd_convertor)
if is_aarch64():
pipeline.add(transform)
pipeline.add(nvosd)
streammux.link(detector)
detector.link(tracker)
tracker.link(nvdsanalytics)
nvdsanalytics.link(tiler)
tiler.link(osd_convertor)
osd_convertor.link(nvosd)
if is_aarch64():
nvosd.link(videosink)
transform.link(videosink)
else:
nvosd.link(videosink)
# take the commandline argument and ensure that it is a uri
if Gst.uri_is_valid(args[1]):
uri = args[1]
else:
uri = Gst.filename_to_uri(args[1])
uridecodebin.set_property('uri', uri)
uridecodebin.connect("pad-added", cb_newpad, streammux.get_request_pad("sink_0"))
GLib.timeout_add(30000, restart_pipeline, pipeline)
# create and event loop and feed gstreamer bus mesages to it
loop = GObject.MainLoop()
bus = pipeline.get_bus()
bus.add_signal_watch()
bus.connect ("message", bus_call, loop)
# start play back and listed to events
pipeline.set_state(Gst.State.PLAYING)
try:
loop.run()
except:
pass
# cleanup
pipeline.set_state(Gst.State.NULL)
if __name__ == '__main__':
sys.exit(main(sys.argv))
You can run it with something like python3 example.py <uri>.
An important detail is: this problem doesn’t exist when running this exact same code, using the exact same model (trafficcamnet), on a Jetson Nano (Jetpack 4.5.1). It’s been running for 5 hours now without crashing.
The code is in python, but I’ve modified the deepstream_test3_app in C to use the same callback function that restarts the pipeline every 30 seconds, and the same problem happens.
• Requirement details( This is for new requirement. Including the module name-for which plugin or for which sample application, the function description)