Jetson Nano: streaming opencv frames via RTSP gstreamer pipeline

[T.F-S]

G’Day all,

I’m attempting to stream processed frames out onto the network via rtsp on a Jetson Nano, while not being too familiar with gstreamer. The code I currently have will execute fine, and vlc will connect to the stream, but no media will appear and vlc appears to be constantly buffering.

Help resolving this issue would be greatly appreciated. (Any help refining the gstreamer pipelines to reduce latency would also be greatly appreciated as that is vitally important and I haven’t managed utilize the hardware encoders/decoders yet.)

Code:

import numpy as np 
import cv2
import keyboard
import gi

gi.require_version('Gst','1.0')
gi.require_version('GstRtspServer','1.0')
from gi.repository import GObject, Gst, GstRtspServer

def gstreamer_pipeline_in():
	return(
		" nvarguscamerasrc sensor-id=0 ! "
		" video/x-raw(memory:NVMM), width=3840, height=2160, format=NV12, framerate=30/1 ! "
		#" nvv4l2h264enc ! "
		" nvvidconv ! "
		" video/x-raw, format=RGBA ! "
		" videoconvert ! appsink "
	)

def gstreamer_pipeline_out():
	return(
		'appsrc is-live=true ! videoconvert ! \
            	 omxh264enc bitrate=12000000 ! video/x-h264, \
             	stream-format=byte-stream ! rtph264pay pt=96 ! \
             	udpsink host=127.0.0.1 port=5400 async=false'
	)

cap = cv2.VideoCapture(gstreamer_pipeline_in(), cv2.CAP_GSTREAMER) 

out = cv2.VideoWriter(gstreamer_pipeline_out(), cv2.CAP_GSTREAMER, 0, 30, (1920,1080), True)

if not (cap.isOpened() and out.isOpened()):
	print(" Gstreamer Pipeline not opened ")
	exit()


rtsp_port_num = 8554

server = GstRtspServer.RTSPServer.new()
server.props.service = "%d" % rtsp_port_num
server.attach(None)

factory = GstRtspServer.RTSPMediaFactory.new()
factory.set_launch("(udpsrc name=pay0 port=5400 buffer-size=524288 \
                caps=\"application/x-rtp, media=video, clock-rate=90000, \
                encoding-name=(string)H264, payload=96 \")")

factory.set_shared(True)
server.get_mount_points().add_factory("/ds-test", factory)

print("Launched streaming")

while(cap.isOpened()):

PROCESSING & CONTROL CODE REMOVED FOR BREVITY ----------------------------------------------------------------

	if(ret == True):
		out.write(frame)

cap.release()
out.release()

[DaneLLL]

Hi,
The pipelines in gstreamer_pipeline_in() and gstreamer_pipeline_out() look wrong. Please refer to the samples and try the pipelines:
Stream processed video with OpenCV on Jetson TX2 - #5 by DaneLLL
OpenvCV, Gstreamer, Python camera capture/access and streaming to RTP

If the issue is still present, you can enable the properties to nvv4l2h264enc for a try:

  insert-aud          : Insert H.264 Access Unit Delimiter(AUD)
                        flags: readable, writable
                        Boolean. Default: false
  insert-sps-pps      : Insert H.264 SPS, PPS at every IDR frame
                        flags: readable, writable
                        Boolean. Default: false
  insert-vui          : Insert H.264 VUI(Video Usability Information) in SPS
                        flags: readable, writable
                        Boolean. Default: false

[T.F-S]

Hi All,

Thanks to u/DaneLLL and a PM, I have managed to get the rtsp stream running. Never managed to get the hardware encoders running though.

The code was ultimately derived from this post: Stream frame opencv via rtsp using gstreamer c++ - #6 by Honey_Patouceul - General Discussion - GStreamer Discourse

Here is the code:

import numpy as np 
import cv2
import keyboard
import gi
from threading import Thread 
import re

gi.require_version('Gst','1.0')
gi.require_version('GstRtspServer','1.0')
from gi.repository import GObject, Gst, GstRtspServer, GLib



Gst.init(None)
serverloop = GLib.MainLoop()
server = GstRtspServer.RTSPServer()
mounts = server.get_mount_points()
factory = GstRtspServer.RTSPMediaFactory()
factory.set_launch('( udpsrc port=5004 ! application/x-rtp,encoding-name=H264 ! rtph264depay ! h264parse ! rtph264pay name=pay0 )')
mounts.add_factory("/test", factory)
server.attach(None)
server_loop_thread = Thread(target=serverloop.run)
print("stream ready at rtsp://127.0.0.1:8554/test")
server_loop_thread.start()




def gstreamer_pipeline_in():
	return(
		" nvarguscamerasrc sensor-id=0 ! "
		" video/x-raw(memory:NVMM), width=3840, height=2160, format=NV12, framerate=30/1 ! "
		#" nvv4l2h264enc ! "
		" nvvidconv ! "
		" video/x-raw, format=RGBA ! "
		" videoconvert ! appsink "
	)

def gstreamer_pipeline_out():
	return(
		'appsrc !  queue !  videoconvert ! video/x-raw,format=I420 ! x264enc key-int-max=30 insert-vui=1 tune=zerolatency ! queue ! h264parse ! rtph264pay ! udpsink host=127.0.0.1 port=5004'
	)


cap = cv2.VideoCapture(gstreamer_pipeline_in(), cv2.CAP_GSTREAMER) 

out = cv2.VideoWriter(gstreamer_pipeline_out(), cv2.CAP_GSTREAMER, 0, 30, (1920,1080), True)

if not (cap.isOpened() and out.isOpened()):
	print(" Gstreamer Pipeline not opened ")
	serverloop.quit()
	exit()


PROCESSING & CONTROL CODE REMOVED FOR BREVITY ----------------------------------------------------------------

while(cap.isOpened()):

	if(ret == True):
		out.write(frame)

cap.release()
out.release()
serverloop.quit()

Thanks for all the help.