Segmentation Fault after setting pipeline to NULL and then PLAYING

• Hardware Platform (Jetson / GPU)
GPUs: RTX 2070S, A100 and V100
• DeepStream Version
5.1 (nvcr.io/nvidia/deepstream:5.1-21.02-devel container)
• TensorRT Version
7.2.2.3
• NVIDIA GPU Driver Version (valid for GPU only)
RTX2070S: 470.63.01
A100: 470.57.02
V100: 455.45.01
• Issue Type( questions, new requirements, bugs)
bug
• How to reproduce the issue ? (This is for bugs. Including which sample app is using, the configuration files content, the command line used and other details for reproducing)

I’m trying to restart the pipeline by setting it to NULL and then to PLAYING when specific problems happen (like when the rtsp server is temporarily unavailable) as one of the implemented ways to keep the service always alive. After the first try to set the pipeline to NULL and then to PLAYING the bounding boxes that should be draw by nvdsosd won’t appear anymore, and after more few tries the app crashes with “Segmentation fault (core dumped)” without throwing errors or warnings just after going to PLAYING state.

This problem only happens when nvinfer plugin is in the pipeline.

I’m using the code above to reproduce the issue on the different machines (using fakesink instead of nveglglessink on A100 and V100). It uses a callback function that restarts the pipeline every 30 seconds.

import sys, json

import gi, pyds, configparser
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst, GLib
from ds_utils.is_aarch_64 import is_aarch64

def bus_call(bus, message, loop):
    t = message.type
    if t == Gst.MessageType.EOS:
        sys.stdout.write("End-of-stream\n")
        loop.quit()
    elif t == Gst.MessageType.ERROR:
        err, debug = message.parse_error()
        sys.stderr.write("Error: %s: %s\n" % (err, debug))
        loop.quit()
    return True

def cb_newpad(decodebin, decoder_src_pad, sinkpad):
    caps=decoder_src_pad.get_current_caps()
    gststruct=caps.get_structure(0)
    gstname=gststruct.get_name()

    features=caps.get_features(0)

    if(gstname.find("video")!=-1):
        decoder_src_pad.link(sinkpad)

def play_if_null(pipeline):
    ret , state, pending = pipeline.get_state(0)
    if ret == Gst.StateChangeReturn.SUCCESS and state == Gst.State.NULL:
        pipeline.set_state(Gst.State.PLAYING)
        return False    
    return True

def restart_pipeline(pipeline):
    state_ret = pipeline.set_state(Gst.State.NULL)
    GLib.timeout_add(1000, play_if_null, pipeline)
    return True

def main(args):
    if len(args) != 2:
        sys.stderr.write("usage: %s <media file or uri>\n" % args[0])
        sys.exit(1)

    GObject.threads_init()
    Gst.init(None)

    pipeline = Gst.Pipeline()
    uridecodebin = Gst.ElementFactory.make("uridecodebin", "sourcebin"); assert uridecodebin
    if not uridecodebin:
        sys.stderr.write("'uridecodebin' gstreamer plugin missing\n")
        sys.exit(1)

    streammux = Gst.ElementFactory.make("nvstreammux", "mux")
    streammux.set_property("width", 640)
    streammux.set_property("height", 480)
    streammux.set_property("batched-push-timeout", 40000)
    streammux.set_property("batch-size", 1)  
    streammux.set_property("live-source", 1)

    detector = Gst.ElementFactory.make("nvinfer", "primary-inference"); assert detector
    tracker = Gst.ElementFactory.make("nvtracker", "tracker"); assert tracker
    nvdsanalytics = Gst.ElementFactory.make("nvdsanalytics", "nvdsanalytics"); assert nvdsanalytics

    # Setting elements properties
    detector.set_property('config-file-path', "/app/config/nvinfer_detector_config.txt")

    pgie_batch_size=detector.get_property("batch-size")

    if(pgie_batch_size != 1):
        print(f"Overriding infer-config batch-size {pgie_batch_size} with max batch size 1")
    detector.set_property("batch-size",1)

    tracker_config = configparser.ConfigParser()
    tracker_config.read("/app/config/nvtracker_tracker_config.txt")
    tracker_config.sections()

    for key in tracker_config['tracker']:
        if key == 'tracker-width' :
            tracker_width = tracker_config.getint('tracker', key)
            tracker.set_property('tracker-width', tracker_width)
        if key == 'tracker-height' :
            tracker_height = tracker_config.getint('tracker', key)
            tracker.set_property('tracker-height', tracker_height)
        if key == 'gpu-id' :
            tracker_gpu_id = tracker_config.getint('tracker', key)
            tracker.set_property('gpu_id', tracker_gpu_id)
        if key == 'll-lib-file' :
            tracker_ll_lib_file = tracker_config.get('tracker', key)
            tracker.set_property('ll-lib-file', tracker_ll_lib_file)
        if key == 'll-config-file' :
            tracker_ll_config_file = tracker_config.get('tracker', key)
            tracker.set_property('ll-config-file', tracker_ll_config_file)
        if key == 'enable-batch-process' :
            tracker_enable_batch_process = tracker_config.getint('tracker', key)
            tracker.set_property('enable_batch_process', tracker_enable_batch_process)

    nvdsanalytics.set_property('config-file', "/app/config/config_nvdsanalytics.txt") 
    nvdsanalytics.set_property('enable', 1)

    tiler = Gst.ElementFactory.make("nvmultistreamtiler", "tiler"); assert tiler
    tiler.set_property("width", 640)
    tiler.set_property("height", 480)
    tiler.set_property("columns",1)
    tiler.set_property("rows",1)

    osd_convertor = Gst.ElementFactory.make("nvvideoconvert", "osd_convertor"); assert osd_convertor
    nvosd = Gst.ElementFactory.make("nvdsosd", "onscreendisplay"); assert nvosd
    nvosd.set_property('process-mode', 1)
    nvosd.set_property('display-text', 1)
    if is_aarch64():
        transform = Gst.ElementFactory.make("nvegltransform", "nvegl-transform"); assert transform
    videosink = Gst.ElementFactory.make("nveglglessink", "videosink"); assert videosink
    videosink.set_property("sync", 0)

    pipeline.add(uridecodebin)
    pipeline.add(streammux)
    pipeline.add(detector)
    pipeline.add(tracker)
    pipeline.add(nvdsanalytics)
    pipeline.add(tiler)
    pipeline.add(videosink)
    pipeline.add(osd_convertor)
    if is_aarch64():
        pipeline.add(transform)
    pipeline.add(nvosd)
    streammux.link(detector)
    detector.link(tracker)
    tracker.link(nvdsanalytics)
    nvdsanalytics.link(tiler)
    tiler.link(osd_convertor) 
    osd_convertor.link(nvosd)
    if is_aarch64():
        nvosd.link(videosink)
        transform.link(videosink)
    else:
        nvosd.link(videosink)

    # take the commandline argument and ensure that it is a uri
    if Gst.uri_is_valid(args[1]):
      uri = args[1]
    else:
      uri = Gst.filename_to_uri(args[1])
    uridecodebin.set_property('uri', uri)

    uridecodebin.connect("pad-added", cb_newpad, streammux.get_request_pad("sink_0"))

    GLib.timeout_add(30000, restart_pipeline, pipeline)

    # create and event loop and feed gstreamer bus mesages to it
    loop = GObject.MainLoop()

    bus = pipeline.get_bus()
    bus.add_signal_watch()
    bus.connect ("message", bus_call, loop)
    
    # start play back and listed to events
    pipeline.set_state(Gst.State.PLAYING)
    try:
      loop.run()
    except:
      pass
    
    # cleanup
    pipeline.set_state(Gst.State.NULL)

if __name__ == '__main__':
    sys.exit(main(sys.argv))

You can run it with something like python3 example.py <uri>.

An important detail is: this problem doesn’t exist when running this exact same code, using the exact same model (trafficcamnet), on a Jetson Nano (Jetpack 4.5.1). It’s been running for 5 hours now without crashing.

The code is in python, but I’ve modified the deepstream_test3_app in C to use the same callback function that restarts the pipeline every 30 seconds, and the same problem happens.

• Requirement details( This is for new requirement. Including the module name-for which plugin or for which sample application, the function description)

Sorry for the late response, is this still an issue to support?

Hello, @kayccc. Thanks for the response. Yes, I’m still having this problem.