Opencv frame as input to deepstream python api pipeline

Accelerated Computing Intelligent Video Analytics DeepStream SDK
9

Hi,

Are you fixed this issue already?
We write an example for Deepstream with OpenCV input.
Hope this can give you more information.

This example is based on the deepstream-test1 python sample.

1. Download python sample

$ git clone https://github.com/NVIDIA-AI-IOT/deepstream_python_apps.git
$ cd deepstream_python_apps/apps/deepstream-test1

2. Apply changes

diff --git a/apps/deepstream-test1/deepstream_test_1.py b/apps/deepstream-test1/deepstream_test_1.py
index 67be84b..5cfb32b 100644
--- a/apps/deepstream-test1/deepstream_test_1.py
+++ b/apps/deepstream-test1/deepstream_test_1.py
@@ -26,12 +26,26 @@ from common.is_aarch_64 import is_aarch64
 from common.bus_call import bus_call
 
 import pyds
+import cv2
 
 PGIE_CLASS_ID_VEHICLE = 0
 PGIE_CLASS_ID_BICYCLE = 1
 PGIE_CLASS_ID_PERSON = 2
 PGIE_CLASS_ID_ROADSIGN = 3
 
+global cap
+
+
+def cb_need_data(appsource, _size, u_data):
+    global cap
+    ret, frame = cap.read()
+    if not ret:
+        appsource.emit("end-of-stream")
+        return
+
+    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGBA)
+    appsource.emit("push-buffer", Gst.Buffer.new_wrapped(frame.tobytes()))
+
 
 def osd_sink_pad_buffer_probe(pad,info,u_data):
     frame_number=0
@@ -127,6 +141,14 @@ def main(args):
         sys.stderr.write("usage: %s <media file or uri>\n" % args[0])
         sys.exit(1)
 
+    global cap
+    cap = cv2.VideoCapture(args[1])
+    width  = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    if not cap.isOpened():
+        print("Can't open input frame. Exiting ...")
+        exit(-1)
+
     # Standard GStreamer initialization
     GObject.threads_init()
     Gst.init(None)
@@ -141,22 +163,19 @@ def main(args):
 
     # Source element for reading from the file
     print("Creating Source \n ")
-    source = Gst.ElementFactory.make("filesrc", "file-source")
-    if not source:
+    appsource = Gst.ElementFactory.make("appsrc", "opencv-source")
+    if not appsource:
         sys.stderr.write(" Unable to create Source \n")
 
-    # Since the data format in the input file is elementary h264 stream,
-    # we need a h264parser
-    print("Creating H264Parser \n")
-    h264parser = Gst.ElementFactory.make("h264parse", "h264-parser")
-    if not h264parser:
-        sys.stderr.write(" Unable to create h264 parser \n")
+    # Use convertor to convert RGBA to NV12
+    nvvideoconvert = Gst.ElementFactory.make("nvvideoconvert","nv-videoconv")
+    if not nvvideoconvert:
+        sys.stderr.write(" Unable to create nvvideoconvert \n")
 
-    # Use nvdec_h264 for hardware accelerated decode on GPU
-    print("Creating Decoder \n")
-    decoder = Gst.ElementFactory.make("nvv4l2decoder", "nvv4l2-decoder")
-    if not decoder:
-        sys.stderr.write(" Unable to create Nvv4l2 Decoder \n")
+    # cap filter
+    caps_filter = Gst.ElementFactory.make("capsfilter","capsfilter")
+    if not caps_filter:
+        sys.stderr.write(" Unable to create caps_filter \n")
 
     # Create nvstreammux instance to form batches from one or more sources.
     streammux = Gst.ElementFactory.make("nvstreammux", "Stream-muxer")
@@ -190,17 +209,21 @@ def main(args):
         sys.stderr.write(" Unable to create egl sink \n")
 
     print("Playing file %s " %args[1])
-    source.set_property('location', args[1])
-    streammux.set_property('width', 1920)
-    streammux.set_property('height', 1080)
+    caps1 = Gst.Caps.from_string("video/x-raw,format=RGBA,width=%d,height=%d,framerate=30/1"%(width, height))
+    caps2 = Gst.Caps.from_string("video/x-raw(memory:NVMM),format=NV12,width=%d,height=%d,framerate=30/1"%(width, height))
+    appsource.set_property('caps', caps1)
+    caps_filter.set_property('caps',caps2)
+
+    streammux.set_property('width', width)
+    streammux.set_property('height', height)
     streammux.set_property('batch-size', 1)
     streammux.set_property('batched-push-timeout', 4000000)
     pgie.set_property('config-file-path', "dstest1_pgie_config.txt")
 
     print("Adding elements to Pipeline \n")
-    pipeline.add(source)
-    pipeline.add(h264parser)
-    pipeline.add(decoder)
+    pipeline.add(appsource)
+    pipeline.add(nvvideoconvert)
+    pipeline.add(caps_filter)
     pipeline.add(streammux)
     pipeline.add(pgie)
     pipeline.add(nvvidconv)
@@ -213,13 +236,13 @@ def main(args):
     # file-source -> h264-parser -> nvh264-decoder ->
     # nvinfer -> nvvidconv -> nvosd -> video-renderer
     print("Linking elements in the Pipeline \n")
-    source.link(h264parser)
-    h264parser.link(decoder)
+    appsource.link(nvvideoconvert)
+    nvvideoconvert.link(caps_filter)
 
     sinkpad = streammux.get_request_pad("sink_0")
     if not sinkpad:
         sys.stderr.write(" Unable to get the sink pad of streammux \n")
-    srcpad = decoder.get_static_pad("src")
+    srcpad = caps_filter.get_static_pad("src")
     if not srcpad:
         sys.stderr.write(" Unable to get source pad of decoder \n")
     srcpad.link(sinkpad)
@@ -238,6 +261,7 @@ def main(args):
     bus.add_signal_watch()
     bus.connect ("message", bus_call, loop)
 
+    appsource.connect("need-data", cb_need_data, "need-data")
     # Lets add probe to get informed of the meta data generated, we add probe to
     # the sink pad of the osd element, since by that time, the buffer would have
     # had got all the metadata.

3. Test

$ export DISPLAY=:0
$ python3 deepstream_test_1.py /opt/nvidia/deepstream/deepstream-6.0/samples/streams/sample_720p.mp4

Thanks.

5 Likes