Getting source frame before nvstreammux

[vino_mm]

Please provide complete information as applicable to your setup.

• Hardware Platform (Jetson / GPU) Jetson
• DeepStream Version 6.1
• JetPack Version (valid for Jetson only) 5.0.2
• TensorRT Version 8.4.1
• Issue Type( questions, new requirements, bugs) Question

I want to get the source frame before nvstreammux since nvstreammux resizes the image.
I put a probe on the sink of nvstreammux and used the below code to fetch the frame.
n_frame = pyds.get_nvds_buf_surface(hash(gst_buffer), 0)
But I’m getting an error saying that only RGBA colour format is supported.

Is there a way to solve this or any other alternate approach I can use to get the frame?

[yuweiw]

We can only support the RGBA color Format now. You can try to add a nvvideoconvert to transform that to the RGBA before the nvstreammux.

[vino_mm]

@yuweiw thank you… It’s working…

def get_source_frame(pad, info, camera_id):
    try:
        gst_buffer = info.get_buffer()
        if not gst_buffer:
            logger.error("Unable to get GstBuffer")
            return

        n_frame = pyds.get_nvds_buf_surface(hash(gst_buffer), 0)
        frame_image = np.array(n_frame, copy=True, order="C")
        frame = cv2.cvtColor(frame_image, cv2.COLOR_RGBA2BGRA)

        _, buffer = cv2.imencode(".png", frame)
        img_bytes = np.array(buffer).tobytes()
        frame_key = "source_frame_" + str(camera_id)
        r.set(frame_key, img_bytes)

    except Exception as err:
        logger.error(f"Error in get_source_frame, Error ==> {err}")

    return Gst.PadProbeReturn.OK

# Create nvvideoconvert plugin
nvvidconv_name = "nvvideoconvert_"
nvvidconv = Gst.ElementFactory.make("nvvideoconvert", nvvidconv_name)
nvvidconv.set_property("nvbuf-memory-type", EDGE_CONFIG.MEMORY_TYPE)
pipeline.add(nvvidconv)
source_bin.link(nvvidconv)

# Creat caps to convert to RGBA
caps_name = "capsfilter_"
caps = Gst.ElementFactory.make("capsfilter", caps_name)
caps.set_property("caps", Gst.Caps.from_string("video/x-raw(memory:NVMM), format=RGBA"))
pipeline.add(caps)
nvvidconv.link(caps)

# Link to streammux
padname = "sink_%u" % camera_id
sinkpad = streammux.get_request_pad(padname)
caps_srcpad = caps.get_static_pad("src")
caps_srcpad.link(sinkpad)

sinkpad.add_probe(Gst.PadProbeType.BUFFER, get_source_frame, int(camera_id))