Hello there!
I am trying to get a deepstream pipeline for optical flow calculation running. The idea is to receive a video stream from a USB camera, do the optical flow calculation and send the resulting image from nvofvisual via RTP stream. The important parts of the Python code look like this:
# Standard GStreamer initialization
Gst.init(None)
# Create GStreamer pipeline
self.pipeline = Gst.Pipeline()
usb_cam_source = Gst.ElementFactory.make("v4l2src", "usb-cam-source")
caps_v4l2src = Gst.ElementFactory.make("capsfilter", "v4l2src_caps")
caps_v4l2src.set_property('caps', Gst.Caps.from_string("video/x-raw, framerate=30/1"))
# Video Converter
vidconvsrc = Gst.ElementFactory.make("videoconvert", "convertor_src1")
nvvidconvsrc = Gst.ElementFactory.make("nvvideoconvert", "convertor_src2")
# Set the caps for nvvidconvsrc
caps_vidconvsrc = Gst.ElementFactory.make("capsfilter", "nvmm_caps")
caps_vidconvsrc.set_property("caps", Gst.Caps.from_string("video/x-raw(memory:NVMM), format=(string)NV12"))
caps_vidconvsink = Gst.ElementFactory.make("capsfilter", "nvmm_caps2")
caps_vidconvsink.set_property("caps", Gst.Caps.from_string("video/x-raw(memory:NVMM), format=(string)NV12"))
# Create nvstreammux instance to form batches from one or more sources.
streammux = Gst.ElementFactory.make("nvstreammux", "Stream-muxer")
streammux.set_property('width', 1216)
streammux.set_property('height', 684)
streammux.set_property('batch-size', 1)
streammux.set_property('batched-push-timeout', 4000000)
nvof = Gst.ElementFactory.make("nvof", "nvopticalflow")
nvof.set_property('preset-level', 0)
nvofvisual = Gst.ElementFactory.make("nvofvisual", "nvopticalflowvisual")
# RTP Stream Out
# Video Converter
nvvidconv2 = Gst.ElementFactory.make("nvvideoconvert", "convertor2")
# Queue
encoder_queue = Gst.ElementFactory.make("queue", "encoder_queue")
encoder_queue.set_property("leaky", 2) # 2 corresponds to 'downstream' leaky queue behavior
encoder_queue.set_property('max-size-buffers', 0)
encoder_queue.set_property('max-size-bytes', 0)
encoder_queue.set_property('max-size-time', 0)
# x264 Encoder
x264enc = Gst.ElementFactory.make('x264enc', 'encoder')
x264enc.set_property("bitrate", 3000)
x264enc.set_property("tune", "zerolatency")
# Create capsfilter element
caps_x264enc = Gst.ElementFactory.make("capsfilter", "caps_x264enc")
caps = Gst.Caps.from_string("video/x-h264,profile=main")
caps_x264enc.set_property("caps", caps)
#H264 Parser
h264parse = Gst.ElementFactory.make("h264parse", "parser")
# RTP Payloader
rtppay = Gst.ElementFactory.make('rtph264pay', 'rtppay')
# UDP Sink
udpsink = Gst.ElementFactory.make('udpsink', 'udpsink')
udpsink.set_property('host', 'IP_ADDRESS') # IP address of the destination machine
udpsink.set_property('port', output_port) # Any port number as per your choice
udpsink.set_property("sync", False)
udpsink.set_property("async", False)
# Add elements to the pipeline
self.get_logger().info('Adding elements to Pipeline')
#USB camera source
self.pipeline.add(usb_cam_source)
self.pipeline.add(caps_v4l2src)
self.pipeline.add(vidconvsrc)
self.pipeline.add(nvvidconvsrc)
self.pipeline.add(caps_vidconvsrc)
self.pipeline.add(streammux)
self.pipeline.add(nvof)
self.pipeline.add(nvofvisual)
self.pipeline.add(nvvidconv2)
self.pipeline.add(encoder_queue)
self.pipeline.add(x264enc)
self.pipeline.add(caps_x264enc)
self.pipeline.add(h264parse)
self.pipeline.add(rtppay)
self.pipeline.add(udpsink)
# Link the elements together
#USB camera source
usb_cam_source.link(caps_v4l2src)
caps_v4l2src.link(vidconvsrc)
vidconvsrc.link(nvvidconvsrc)
nvvidconvsrc.link(caps_vidconvsrc)
sinkpad = streammux.get_request_pad("sink_0")
srcpad = caps_vidconvsrc.get_static_pad("src")
srcpad.link(sinkpad)
streammux.link(nvof)
nvof.link(nvofvisual)
nvofvisual.link(nvvidconv2)
nvvidconv2.link(encoder_queue)
encoder_queue.link(x264enc)
x264enc.link(caps_x264enc)
caps_x264enc.link(h264parse)
h264parse.link(rtppay)
rtppay.link(udpsink)
# Create an event loop and feed gstreamer bus messages to it
self.loop = GLib.MainLoop()
bus = self.pipeline.get_bus()
bus.add_signal_watch()
bus.connect("message", bus_call, self.loop)
When I run the pipeline, I get the following error:
Error: gst-stream-error-quark: Can not initialize x264 encoder. (8): gstx264enc.c(1898): gst_x264_enc_init_encoder (): /GstPipeline:pipeline0/GstX264Enc:encoder
When I run the pipeline without the nvofvisual element, i.e. doing
streammux.link(nvof)
nvof.link(nvvidconv2)
nvvidconv2.link(encoder_queue)
I do receive the camera video via the RTP stream, but (obviously) without the visualization of the optical flow.
Furthermore, using the nvinfer, nvtracker and nvosd elements in the same pipeline (instead of nvof and nvofvisual), like this:
streammux.link(pgie)
pgie.link(tracker)
tracker.link(nvvidconv)
nvvidconv.link(nvosd)
nvosd.link(nvvidconv2)
nvvidconv2.link(encoder_queue)
works as expected and I do receive the video with bounding boxes via the RTP stream.
I do not understand, why I get this error. From the deepstream documentation, it seems that the output types of nvosd and nvofvisual (RGBA output buffer) are the same and I should be able to exchange the pgie related and optical flow related elements in the pipeline.
I am happy to receive any input on what I could try to get this pipeline working!
Here are some specs of the system I use:
• Hardware Platform (Jetson / GPU): Jetson Orin Nano, 8GB
• DeepStream Version: 6.3
• JetPack Version (valid for Jetson only): 5.1.2
• TensorRT Version: 8.5.2.2
