Deepstream python Error, SYNC_IOC_FENCE_INFO ioctl failed with 9

Please provide complete information as applicable to your setup.

• Hardware Platform (Jetson / GPU)
• DeepStream Version
• JetPack Version (valid for Jetson only)
• TensorRT Version
• NVIDIA GPU Driver Version (valid for GPU only)
• Issue Type( questions, new requirements, bugs)
• How to reproduce the issue ? (This is for bugs. Including which sample app is using, the configuration files content, the command line used and other details for reproducing)
• Requirement details( This is for new requirement. Including the module name-for which plugin or for which sample application, the function description)

Running a deepstream 6.0 python app for object detection on jetson nano, below is themodified rtsp sink app

def main(args):

    # Standard GStreamer initialization
    GObject.threads_init()
    Gst.init(None)

    # Create gstreamer elements
    # Create Pipeline element that will form a connection of other elements
    print("Creating Pipeline \n ")
    pipeline = Gst.Pipeline()
    if not pipeline:
        sys.stderr.write(" Unable to create Pipeline \n")

    # Source element for reading from USB webcam
    print("Creating Source \n ")
    source = Gst.ElementFactory.make("v4l2src", "usb-cam-source")
    if not source:
        sys.stderr.write(" Unable to create Source \n")

    caps_v4l2src = Gst.ElementFactory.make("capsfilter", "v4l2src_caps")
    if not caps_v4l2src:
        sys.stderr.write(" Unable to create v4l2src capsfilter \n")


    print("Creating Video Converter \n")

    # Adding videoconvert -> nvvideoconvert as not all
    # raw formats are supported by nvvideoconvert;
    # Say YUYV is unsupported - which is the common
    # raw format for many logi usb cams
    # In case we have a camera with raw format supported in
    # nvvideoconvert, GStreamer plugins' capability negotiation
    # shall be intelligent enough to reduce compute by
    # videoconvert doing passthrough (TODO we need to confirm this)


    # videoconvert to make sure a superset of raw formats are supported
    vidconvsrc = Gst.ElementFactory.make("videoconvert", "convertor_src1")
    if not vidconvsrc:
        sys.stderr.write(" Unable to create videoconvert \n")

    # nvvideoconvert to convert incoming raw buffers to NVMM Mem (NvBufSurface API)
    nvvidconvsrc = Gst.ElementFactory.make("nvvideoconvert", "convertor_src2")
    if not nvvidconvsrc:
        sys.stderr.write(" Unable to create Nvvideoconvert \n")

    caps_vidconvsrc = Gst.ElementFactory.make("capsfilter", "nvmm_caps")
    if not caps_vidconvsrc:
        sys.stderr.write(" Unable to create capsfilter \n")

    # Create nvstreammux instance to form batches from one or more sources.
    streammux = Gst.ElementFactory.make("nvstreammux", "Stream-muxer")
    if not streammux:
        sys.stderr.write(" Unable to create NvStreamMux \n")

    # Use nvinfer to run inferencing on decoder's output,
    # behaviour of inferencing is set through config file
    pgie = Gst.ElementFactory.make("nvinfer", "primary-inference")
    if not pgie:
        sys.stderr.write(" Unable to create pgie \n")
    
    # Use convertor to convert from NV12 to RGBA as required by nvosd
    nvvidconv = Gst.ElementFactory.make("nvvideoconvert", "convertor")
    if not nvvidconv:
        sys.stderr.write(" Unable to create nvvidconv \n")
    
    # Create OSD to draw on the converted RGBA buffer
    nvosd = Gst.ElementFactory.make("nvdsosd", "onscreendisplay")
    if not nvosd:
        sys.stderr.write(" Unable to create nvosd \n")

    nvvidconv_postosd = Gst.ElementFactory.make("nvvideoconvert", "convertor_postosd")
    if not nvvidconv_postosd:
        sys.stderr.write(" Unable to create nvvidconv_postosd \n")

    # Create a caps filter
    caps = Gst.ElementFactory.make("capsfilter", "filter")
    caps.set_property("caps", Gst.Caps.from_string("video/x-raw(memory:NVMM), format=I420"))

    # Make the encoder
    if codec == "H264":
        encoder = Gst.ElementFactory.make("nvv4l2h264enc", "encoder")
        print("Creating H264 Encoder")
    elif codec == "H265":
        encoder = Gst.ElementFactory.make("nvv4l2h265enc", "encoder")
        print("Creating H265 Encoder")
    if not encoder:
        sys.stderr.write(" Unable to create encoder")
    encoder.set_property('bitrate', bitrate)
    if is_aarch64():
        encoder.set_property('preset-level', 1)
        encoder.set_property('insert-sps-pps', 1)
        encoder.set_property('bufapi-version', 1)
    
    # Make the payload-encode video into RTP packets
    if codec == "H264":
        rtppay = Gst.ElementFactory.make("rtph264pay", "rtppay")
        print("Creating H264 rtppay")
    elif codec == "H265":
        rtppay = Gst.ElementFactory.make("rtph265pay", "rtppay")
        print("Creating H265 rtppay")
    if not rtppay:
        sys.stderr.write(" Unable to create rtppay")
    
    # Make the UDP sink
    updsink_port_num = 5400
    sink = Gst.ElementFactory.make("udpsink", "udpsink")
    if not sink:
        sys.stderr.write(" Unable to create udpsink")
    
    sink.set_property('host', '224.224.255.255')
    sink.set_property('port', updsink_port_num)
    sink.set_property('async', False)
    sink.set_property('sync', 1)
    
    print("Playing cam %s " %camera_path)
    caps_v4l2src.set_property('caps', Gst.Caps.from_string("video/x-raw, framerate=30/1"))
    caps_vidconvsrc.set_property('caps', Gst.Caps.from_string("video/x-raw(memory:NVMM)"))
    source.set_property('device', camera_path)
    streammux.set_property('width', 1280)
    streammux.set_property('height', 720)
    streammux.set_property('batch-size', 1)
    streammux.set_property('batched-push-timeout', 4000000)
    # Construct the configuration file path with the specified "graintype" prefix
    config_file_path = f"{graintype}_config_infer_primary_yoloV5.txt"
    print(config_file_path)
    
    # Set the property with the constructed configuration file path
    pgie.set_property('config-file-path', config_file_path)

    print("Adding elements to Pipeline \n")
    pipeline.add(source)
    pipeline.add(caps_v4l2src)
    pipeline.add(vidconvsrc)
    pipeline.add(nvvidconvsrc)
    pipeline.add(caps_vidconvsrc)
    pipeline.add(streammux)
    pipeline.add(pgie)
    pipeline.add(nvvidconv)
    pipeline.add(nvosd)
    pipeline.add(nvvidconv_postosd)
    pipeline.add(caps)
    pipeline.add(encoder)
    pipeline.add(rtppay)
    pipeline.add(sink)

    # we link the elements together
    # v4l2src -> nvvideoconvert -> mux -> 
    # nvinfer -> nvvideoconvert -> nvosd --> nvvidconv_postosd -> 
    # caps -> encoder -> rtppay -> udpsink

    print("Linking elements in the Pipeline \n")
    source.link(caps_v4l2src)
    caps_v4l2src.link(vidconvsrc)
    vidconvsrc.link(nvvidconvsrc)
    nvvidconvsrc.link(caps_vidconvsrc)

    sinkpad = streammux.get_request_pad("sink_0")
    if not sinkpad:
        sys.stderr.write(" Unable to get the sink pad of streammux \n")

    srcpad = caps_vidconvsrc.get_static_pad("src")
    if not srcpad:
        sys.stderr.write(" Unable to get source pad of caps_vidconvsrc \n")
    srcpad.link(sinkpad)
    streammux.link(pgie)
    pgie.link(nvvidconv)
    nvvidconv.link(nvosd)
    nvosd.link(nvvidconv_postosd)
    nvvidconv_postosd.link(caps)
    caps.link(encoder)
    encoder.link(rtppay)
    rtppay.link(sink)

    # create an event loop and feed gstreamer bus mesages to it
    global loop 
    loop = GObject.MainLoop()
    bus = pipeline.get_bus()
    bus.add_signal_watch()
    bus.connect ("message", bus_call, loop)
    
    # Start streaming
    rtsp_port_num = 8554
    #rtsp_address = "20.204.93.170"
    rtsp_address = "0.0.0.0"

    
    server = GstRtspServer.RTSPServer.new()
    server.set_address(rtsp_address)

    server.props.service = "%d" % rtsp_port_num
    server.attach(None)
    factory = GstRtspServer.RTSPMediaFactory.new()
    #factory.set_launch("( rtspsrc location=rtsp://%s:%d/%s ! rtph264depay ! h264parse ! x264enc ! rtph264pay name=pay0 pt=96 )"
    #                " udpsink host=%s port=%d" % (rtsp_address, rtsp_port_num, path, rtsp_address, updsink_port_num))

    factory.set_launch( "( udpsrc name=pay0 port=%d buffer-size=524288 caps=\"application/x-rtp, media=video, clock-rate=90000, encoding-name=(string)%s, payload=96 \" )" % (updsink_port_num, codec))
    factory.set_shared(True)
    server.get_mount_points().add_factory("/rgrt", factory)
    
    print("\n *** DeepStream: Launched RTSP Streaming at rtsp://localhost:%d/rgrt ***\n\n" % rtsp_port_num)
    
    # Lets add probe to get informed of the meta data generated, we add probe to
    # the sink pad of the osd element, since by that time, the buffer would have
    # had got all the metadata.
    osdsinkpad = nvosd.get_static_pad("sink")
    if not osdsinkpad:
        sys.stderr.write(" Unable to get sink pad of nvosd \n")
    
    osdsinkpad.add_probe(Gst.PadProbeType.BUFFER, osd_sink_pad_buffer_probe, 0)
    

    # Register the signal handler for SIGTERM
    signal.signal(signal.SIGTERM, sigterm_handler)

    # Add a GPIO event callback
    GPIO.add_event_detect(GPIO_PIN, GPIO.FALLING, callback=gpio_callback, bouncetime=200)


    # Create and start the GMainLoop
    #loop = GObject.MainLoop()
    print("Starting pipeline \n")
    pipeline.set_state(Gst.State.PLAYING)
    try:
        loop.run()
    except KeyboardInterrupt:
        stop_and_cleanup(pipeline)
    except Exception as e:
        print(f"An error occurred: {str(e)}")
        stop_and_cleanup(pipeline)
        GPIO.cleanup()
    # cleanup
    pipeline.set_state(Gst.State.NULL)
    GPIO.cleanup()

def parse_args():
    parser = argparse.ArgumentParser(description='RTSP Output From USB Webcam Custom Application Help ')
    parser.add_argument("-i", "--input",
                  help="Path to input /dev/video* device", required=True)
    parser.add_argument("-c", "--codec", default="H264",
                  help="RTSP Streaming Codec H264/H265 , default=H264", choices=['H264','H265'])
    parser.add_argument("-b", "--bitrate", default=4000000,
                  help="Set the encoding bitrate ", type=int)
    parser.add_argument("--graintype", default="chana",
                  help="Prefix for configuration file (graintype)", type=str)  # New argument for config prefix
    # Check input arguments
    if len(sys.argv)==1:
        parser.print_help(sys.stderr)
        sys.exit(1)
    args = parser.parse_args()
    global codec
    global bitrate
    global camera_path
    global graintype
    global csv_file_name
    global experiment_id
    codec = args.codec
    bitrate = args.bitrate
    camera_path = args.input
    graintype = args.graintype
    experiment_id = read_experiment_id()
    #csv_timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')
    #csv_file_name = create_csv_file(csv_timestamp, graintype)

    return 0

if __name__ == '__main__':
    parse_args()
    pipeline = None  # Declare pipeline here so it's accessible in the signal handler
    sys.exit(main(sys.argv))

The app is able to inference and give output over a rtsp server correctly, but after processing 458 frames it stops, with the following error

Frame Number=458 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
SYNC_IOC_FENCE_INFO ioctl failed with 9
0:03:36.703433577 17129     0x14d3a2d0 WARN                 nvinfer gstnvinfer.cpp:1376:convert_batch_and_push_to_input_thread:<primary-inference> error: NvBufSurfTransform failed with error -2 while converting buffer
Error: gst-stream-error-quark: NvBufSurfTransform failed with error -2 while converting buffer (1): /dvs/git/dirty/git-master_linux/deepstream/sdk/src/gst-plugins/gst-nvinfer/gstnvinfer.cpp(1376): convert_batch_and_push_to_input_thread (): /GstPipeline:pipeline0/GstNvInfer:primary-inference
Frame Number=459 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable 'obj_meta' referenced before assignment
0:03:37.557729022 17129     0x14d3a2d0 WARN                 nvinfer gstnvinfer.cpp:1376:convert_batch_and_push_to_input_thread:<primary-inference> error: NvBufSurfTransform failed with error -3 while converting buffer
Segmentation fault (core dumped)

Please provide me with a solution to resolve it ?

  NvBufSurfTransformError_Invalid_Params = -3,
  /** Specifies a runtime execution error. */
  NvBufSurfTransformError_Execution_Error = -2,

I guess this issue is related with the nvvideoconvert plugin.

Maybe it has something to do with your camera output.

Can you provide a detail log with GST_DEBUG=3 python3 your_app ?

You can get the callstack with the following command line when the app crash.

gdb -ex run --args python3 your_app.py

Another tips, Can you reproduce this problem by simulating the output of v4l2src
through videotestsrc?

Then I can look into it.

this is the tail of the out put

s:976:gst_v4l2src_create: lost frames detected: count = 6 - ts: 0:03:09.157424045
Frame Number=434 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:24.716681396 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:09.557418358
Frame Number=435 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:25.133791197 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:09.957444566
Frame Number=436 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:25.555823283 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 6 - ts: 0:03:10.425457368
Frame Number=437 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:25.974475592 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:10.829366743
Frame Number=438 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:26.393331905 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 6 - ts: 0:03:11.297435264
Frame Number=439 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:26.946229531 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:11.697347618
Frame Number=440 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:27.557277452 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:12.097390149
Frame Number=441 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:27.740856799 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:12.501307202
Frame Number=442 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:28.229399569 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 6 - ts: 0:03:12.969325993
Frame Number=443 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:28.603050040 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 7 - ts: 0:03:13.505283774
Frame Number=444 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:29.127081144 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 8 - ts: 0:03:14.105264284
Frame Number=445 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:29.575151353 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 2 - ts: 0:03:14.305311566
Frame Number=446 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:29.994788742 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 6 - ts: 0:03:14.773297889
Frame Number=447 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:30.432164300 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:15.177227535
Frame Number=448 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:30.854508952 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 7 - ts: 0:03:15.709322514
Frame Number=449 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:31.282510946 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:16.113199461
Frame Number=450 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:31.744683160 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:16.513244202
Frame Number=451 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:32.174253808 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 6 - ts: 0:03:16.981195410
Frame Number=452 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:32.628817498 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:17.381220316
Frame Number=453 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:33.045771335 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 6 - ts: 0:03:17.853142201
Frame Number=454 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:33.468043957 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 6 - ts: 0:03:18.321133253
Frame Number=455 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:33.889231420 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:18.721137097
Frame Number=456 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:34.307840718 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 6 - ts: 0:03:19.189122889
Frame Number=457 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:34.735606724 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:19.589151097
Frame Number=458 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:35.206995257 25476 0x2898c8a0 WARN v4l2src gstv4l2src.c:976:gst_v4l2src_create: lost frames detected: count = 5 - ts: 0:03:19.993095409
Error sending telemetry to Azure IoT Hub: Unexpected failure
Frame Number=459 Total Number of Objects=0 Impurity_Counts=0 FM_Count=0
Error saving frames: local variable ‘obj_meta’ referenced before assignment
0:03:35.271161165 25476 0x289f3a80 ERROR nvvideoconvert gstnvvideoconvert.c:3488:gst_nvvideoconvert_transform: buffer transform failed
0:03:35.271369285 25476 0x289f3a80 WARN nvinfer gstnvinfer.cpp:2288:gst_nvinfer_output_loop: error: Internal data stream error.
0:03:35.271417201 25476 0x289f3a80 WARN nvinfer gstnvinfer.cpp:2288:gst_nvinfer_output_loop: error: streaming stopped, reason error (-5)
Error: gst-stream-error-quark: Internal data stream error. (1): /dvs/git/dirty/git-master_linux/deepstream/sdk/src/gst-plugins/gst-nvinfer/gstnvinfer.cpp(2288): gst_nvinfer_output_loop (): /GstPipeline:pipeline0/GstNvInfer:primary-inference:
streaming stopped, reason error (-5)

log for` “gdb -ex run --args python3 your_app.py”
logs1.txt (34.3 KB)

`

Azure IOT hub connection is creating a issue here

There is no update from you for a period, assuming this is not an issue anymore. Hence we are closing this topic. If need further support, please open a new one. Thanks

Is this issue sovled ?

Does the deepstream problem still exist?

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.