Appsrc with numpy input in Python

Hi, I am using deepstream 4, and i need to use appsrc, the following is the pipeline i will be using

# Standard GStreamer initialization
GObject.threads_init()
Gst.init(None)

# Create gstreamer elements
# Create Pipeline element that will form a connection of other elements
print("Creating Pipeline \n ")
pipeline = Gst.Pipeline()

if not pipeline:
    sys.stderr.write(" Unable to create Pipeline \n")

# Source element for reading from the file
print("Creating Source \n ")
appsource = Gst.ElementFactory.make("appsrc", "opencv-source")
if not appsource:
    sys.stderr.write(" Unable to create Source \n")

videoconvert = Gst.ElementFactory.make("videoconvert","videoconver")
if not videoconvert:
    sys.stderr.write(" error videoconvert")

nvvideoconvert = Gst.ElementFactory.make("nvvideoconvert","nv-videoconv")
if not nvvideoconvert:
    sys.stderr.write(" error nvvid1")

nvstreammux = Gst.ElementFactory.make("nvstreammux", "nv-streammux")
if not nvstreammux:
    sys.stderr.write(" unable-streammux")

   
nvvideoconvert2 = Gst.ElementFactory.make("nvvideoconvert","nv-videoconv2")
if not nvvideoconvert2:
      sys.stderr.write(" unable-streammux")

transform = Gst.ElementFactory.make("nvegltransform", "nvegl-transform")

sink = Gst.ElementFactory.make("nveglglessink", "nvvideo-renderer")
if not sink:
    sys.stderr.write(" Unable to create egl sink \n")


caps = Gst.Caps.from_string("video/x-raw,format=BGR,width=640,height=480,framerate=30/1")
appsource.set_property('caps', caps)
nvstreammux.set_property('width', 640)
nvstreammux.set_property('height', 480)
nvstreammux.set_property('batch-size', 1)
nvstreammux.set_property('batched-push-timeout', 4000000)

print("Adding elements to Pipeline \n")
pipeline.add(appsource)
pipeline.add(videoconvert)
pipeline.add(nvvideoconvert)
pipeline.add(nvstreammux)
pipeline.add(nvvideoconvert2)
pipeline.add(transform)
pipeline.add(sink)

# Working pipeline
#print("Linking elements in the Pipeline \n")
#appsource.link(videoconvert)
#videoconvert.link(nvvideoconvert)
#nvvideoconvert.link(transform)
#transform.link(sink)

# we link the elements together
print("Linking elements in the Pipeline \n")
appsource.link(videoconvert)
videoconvert.link(nvvideoconvert)
sinkpad = nvstreammux.get_request_pad("sink_0")
if not sinkpad:
    sys.stderr.write(" Unable to get the sink pad of streammux \n")
srcpad = nvvideoconvert.get_static_pad("src")
if not srcpad:
    sys.stderr.write(" Unable to get src pad of nvvideoconvert")
srcpad.link(sinkpad)    
nvstreammux.link(nvvideoconvert2)
nvvideoconvert2.link(transform)
transform.link(sink)

# create an event loop and feed gstreamer bus mesages to it
loop = GObject.MainLoop()
bus = pipeline.get_bus()
bus.add_signal_watch()
bus.connect ("message", bus_call, loop)


# start play back and listen to events
print("Starting pipeline \n")
pipeline.set_state(Gst.State.PLAYING)

# Push buffer and check
for _ in range(1):
    arr = np.random.randint(low=0,high=255,size=(480,640,3),dtype=np.uint8)
    appsource.emit("push-buffer", ndarray_to_gst_buffer(arr))
    time.sleep(0.3)
appsource.emit("end-of-stream")
try:
    loop.run()
except:
    pass
# cleanup
pipeline.set_state(Gst.State.NULL)

I am facing this error -

**Using winsys: x11
0:00:00.861759706 7781 0x4120fb70 FIXME default gstutils.c:3981:gst_pad_create_stream_id_internal:opencv-source:src Creating random stream-id, consider implementing a deterministic way of creating a stream-id
0:00:00.993175510 7781 0x4120fb70 WARN nvstreammux gstnvstreammux.c:309:gst_nvstreammux_chain: error: Input buffer number of surfaces (-10565357) must be equal to mux->num_surfaces_per_frame (1)
Set nvstreammux property num-surfaces-per-frame appropriately

0:00:00.993318586 7781 0x4120fb70 WARN basesrc gstbasesrc.c:3055:gst_base_src_loop: error: Internal data stream error.
0:00:00.993348795 7781 0x4120fb70 WARN basesrc gstbasesrc.c:3055:gst_base_src_loop: error: streaming stopped, reason error (-5)
Error: gst-stream-error-quark: Input buffer number of surfaces (-10565357) must be equal to mux->num_surfaces_per_frame (1)
Set nvstreammux property num-surfaces-per-frame appropriately
(1): /dvs/git/dirty/git-master_linux/deepstream/sdk/src/gst-plugins/gst-nvmultistream/gstnvstreammux.c(309): gst_nvstreammux_chain (): /GstPipeline:pipeline0/GstNvStreamMux:nv-streammux
**

Hi,
We have python reference samples:


Please check if you can run deepstream-test1 first. The test1 app is close to your usecase ans easier to start the customization.

Hi @DaneLLL,
I have run the Sample SDK and it works fine, no issues there. My pipeline requires numpy as an input. The posted above is just a trial of the functionaltiy. appsrc -> videoconvert -> nvvideoconvert -> transform -> eglsink works fine. Did not have any issue. I am not able to connect nvstreammux in between.
The above program is running on a jetson-nano board. I have derived that from deepstream-test1.

Hi,
Here is a sample of changing to v4l2src in deepstream-test1:


Although it is in C code, you may take a loo and check ig you can apply similar modification in python.

Hi,
In my use case, i needed to work with appsrc and numpy (numpy image passed on with a third-party application). However i got it it with work with numpy, here is a sample code to make it work.

#!/usr/bin/env python
import json
import logging
import cv2


import sys
sys.path.append('../')
import gi
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst
from common.is_aarch_64 import is_aarch64
from common.bus_call import bus_call
import numpy as np
import pyds
import time

def ndarray_to_gst_buffer(array: np.ndarray) -> Gst.Buffer:
    """Converts numpy array to Gst.Buffer"""
    return Gst.Buffer.new_wrapped(array.tobytes())


def main():
    # Standard GStreamer initialization
    GObject.threads_init()
    Gst.init(None)

    # Create Pipeline element that will form a connection of other elements
    print("Creating Pipeline \n ")
    pipeline = Gst.Pipeline()

    if not pipeline:
        sys.stderr.write(" Unable to create Pipeline \n")

    # Source element for reading from the file
    print("Creating Source \n ")
    appsource = Gst.ElementFactory.make("appsrc", "numpy-source")
    if not appsource:
        sys.stderr.write(" Unable to create Source \n")

    nvvideoconvert = Gst.ElementFactory.make("nvvideoconvert","nv-videoconv")
    if not nvvideoconvert:
        sys.stderr.write(" error nvvid1")

    caps_filter = Gst.ElementFactory.make("capsfilter","capsfilter1")
    if not caps_filter:
        sys.stderr.write(" error capsf1")

    transform = Gst.ElementFactory.make("nvegltransform", "nvegl-transform")

    sink = Gst.ElementFactory.make("nveglglessink", "nvvideo-renderer")
    if not sink:
        sys.stderr.write(" Unable to create egl sink \n")

    caps_in = Gst.Caps.from_string("video/x-raw,format=RGBA,width=640,height=480,framerate=30/1")
    caps = Gst.Caps.from_string("video/x-raw(memory:NVMM),format=NV12,width=640,height=480,framerate=30/1")
    appsource.set_property('caps', caps_in)
    caps_filter.set_property('caps',caps)

    print("Adding elements to Pipeline \n")
    pipeline.add(appsource)
    pipeline.add(nvvideoconvert)
    pipeline.add(caps_filter)
    pipeline.add(transform)
    pipeline.add(sink)
    
    # Working Link pipeline
    print("Linking elements in the Pipeline \n")
    appsource.link(nvvideoconvert)
    nvvideoconvert.link(caps_filter)
    caps_filter.link(transform)
    transform.link(sink)

    # create an event loop and feed gstreamer bus mesages to it
    loop = GObject.MainLoop()
    bus = pipeline.get_bus()
    bus.add_signal_watch()
    bus.connect ("message", bus_call, loop)


    # start play back and listen to events
    print("Starting pipeline \n")
    pipeline.set_state(Gst.State.PLAYING)

    # Push buffer and check
    for _ in range(10):
        arr = np.random.randint(low=0,high=255,size=(480,640,3),dtype=np.uint8)
        arr = cv2.cvtColor(arr, cv2.COLOR_BGR2RGBA)
        appsource.emit("push-buffer", ndarray_to_gst_buffer(arr))
        time.sleep(0.3)
    appsource.emit("end-of-stream")
    try:
        loop.run()
    except:
        pass
    # cleanup
    pipeline.set_state(Gst.State.NULL)

if __name__ == '__main__':
    sys.exit(main())

Hi,
Many thanks for the sharing.