Memory leak while resetting the stream

• Hardware Platform (Jetson / GPU) Jetson
• DeepStream Version 6.1
• JetPack Version (valid for Jetson only) 5.0.1
• TensorRT Version 8.4.1
• Issue Type( questions, new requirements, bugs) Bugs

hello, we adapted the code from runtime_source_add_delete to reset our streams (removing and adding again).

def remove_source(camera_id, disable=False):
    global g_source_bin_dict
    global pipeline
    global streammux
    global STREAM_COUNT

    try:
        if camera_status.get(camera_id) != "DELETE":
            camera_status[camera_id] = "DELETE"

            source_id = camera_id

            # Attempt to change status of source to be released
            state_return = g_source_bin_dict[source_id].set_state(Gst.State.NULL)

            if state_return == Gst.StateChangeReturn.SUCCESS:
                # Set nvvidconv & caps to null
                nvvidconv_name = f"spotgenius_nvvideoconvert_{camera_id}"
                nvvidconv = pipeline.get_by_name(nvvidconv_name)
                nvvidconv.set_state(Gst.State.NULL)

                caps_name = f"spotgenius_capsfilter_{camera_id}"
                caps = pipeline.get_by_name(caps_name)
                caps.set_state(Gst.State.NULL)

                pad_name = "sink_%u" % source_id
                # Retrieve sink pad to be released
                sinkpad = streammux.get_static_pad(pad_name)
                # Send flush stop event to the sink pad, then release from the streammux
                sinkpad.send_event(Gst.Event.new_flush_stop(False))
                streammux.release_request_pad(sinkpad)

                # Remove the source bin from the pipeline
                pipeline.remove(g_source_bin_dict[source_id])

                # Remove nvvidconv & caps from the pipeline
                pipeline.remove(nvvidconv)
                pipeline.remove(caps)

                del post_process_dict[camera_id]
                del camera_last_update_time[camera_id]

                if disable:
                    del camera_frame_count[camera_id]
                    del camera_frame_total_resets[camera_id]
                    del bad_frame_counter[camera_id]
                    del previous_frame_count[camera_id]
                    del output_fps[camera_id]
                    del decode_previous_frames_count[camera_id]
                    del decode_frames_count[camera_id]
                    del decode_fps[camera_id]
                    del camera_resolution[camera_id]
                    del online_offline_dict[camera_id]
                    del camera_timestamp[camera_id]
                    del camera_status[camera_id]
                    del stream_reset_counter[camera_id]
                    del previous_time[camera_id]
                    del num_cars_count[camera_id]
                    del camera_check_status[camera_id]

                STREAM_COUNT -= 1

                logger.info(
                    f"Removing stream from pipeline ==> STREAM_COUNT = {STREAM_COUNT} && Camera ID = {camera_id}"
                )

            elif state_return == Gst.StateChangeReturn.FAILURE:
                logger.error(
                    f"Failed to remove stream from pipeline ==> STREAM_COUNT = {STREAM_COUNT} && Camera ID = {camera_id}"
                )

            elif state_return == Gst.StateChangeReturn.ASYNC:
                logger.error("Async Change Return @ remove_source")

            time.sleep(0.1)
    except Exception as err:
        logger.error(f"Error in remove_source, Error ==> {err}")

    return True


def add_source(camera_id, stream_queue, enable=False):
    global g_source_bin_dict
    global pipeline
    global STREAM_COUNT
    global streammux
    global camera_status

    try:
        if camera_status.get(camera_id) == "CREATE":
            remove_source(camera_id)

        camera_status[camera_id] = "CREATE"

        source_id = camera_id

        stream_dict = stream_queue.get("stream_dict")
        stream_url = stream_dict.get("stream_url")

        source_bin = create_source_bin(source_id, stream_url)
        g_source_bin_dict[source_id] = source_bin
        pipeline.add(source_bin)

        srcpad = source_bin.get_static_pad("src")
        srcpad.add_probe(
            Gst.PadProbeType.BUFFER, source_bin_buffer_probe, int(camera_id)
        )

        # Create nvvideoconvert plugin
        nvvidconv_name = f"spotgenius_nvvideoconvert_{camera_id}"
        nvvidconv = Gst.ElementFactory.make("nvvideoconvert", nvvidconv_name)
        nvvidconv.set_property("nvbuf-memory-type", EDGE_CONFIG.MEMORY_TYPE)
        pipeline.add(nvvidconv)
        source_bin.link(nvvidconv)

        # Creat caps to convert to RGBA
        caps_name = f"spotgenius_capsfilter_{camera_id}"
        caps = Gst.ElementFactory.make("capsfilter", caps_name)
        caps.set_property(
            "caps", Gst.Caps.from_string("video/x-raw(memory:NVMM), format=RGBA")
        )
        pipeline.add(caps)
        nvvidconv.link(caps)

        # Link to streammux
        padname = "sink_%u" % camera_id
        sinkpad = streammux.get_request_pad(padname)
        caps_srcpad = caps.get_static_pad("src")
        caps_srcpad.link(sinkpad)

        # srcpad.link(sinkpad)
        sinkpad.add_probe(Gst.PadProbeType.BUFFER, get_source_frame, int(camera_id))

        state_return = g_source_bin_dict[source_id].set_state(Gst.State.PLAYING)

        if state_return == Gst.StateChangeReturn.SUCCESS:
            # set nvvidconv & caps to playing
            nvvidconv.set_state(Gst.State.PLAYING)
            caps.set_state(Gst.State.PLAYING)

            post_process_dict[camera_id] = stream_queue
            camera_last_update_time[camera_id] = datetime.datetime.utcnow()
            if enable:
                camera_frame_count[camera_id] = 0
                camera_frame_total_resets[camera_id] = 0
                bad_frame_counter[camera_id] = 0
                previous_frame_count[camera_id] = 0
                output_fps[camera_id] = 0
                camera_check_status[camera_id] = "OFFLINE"


            STREAM_COUNT += 1
            logger.info(
                f"Adding stream to pipeline ==> STREAM_COUNT = {STREAM_COUNT} && Camera ID = {camera_id}"
            )

        elif state_return == Gst.StateChangeReturn.FAILURE:
            logger.error(
                f"Failed to add stream to pipeline ==> STREAM_COUNT = {STREAM_COUNT} && Camera ID = {camera_id}"
            )

        elif state_return == Gst.StateChangeReturn.ASYNC:
            logger.error("Async Change Return @ add_source")

    except Exception as err:
        logger.error(f"Error in add source, Error ==> {err}")

    return True

Seems like there is a memory buildup happening each time when this operation is happening.

  1. what is your device model? can you use the latest Deepstream version? could you narrow down issue and highlight the code? which new code will cause memory leak?
  2. if you still find memory leak, please refer to this topic . can you provide more logs by nvmemstat ? it can save the memory usage at intervals. can you provide more valgrind logs by this method vagrid , it can give memory details.

There is no update from you for a period, assuming this is not an issue anymore. Hence we are closing this topic. If need further support, please open a new one. Thanks

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.