Setup 1
• Hardware Platform (GPU) NVIDIA RTX 3500 Ada
• DeepStream Version 7.1
• TensorRT Version 10.5.0.18
• NVIDIA GPU Driver Version (valid for GPU only) 553.46
• Issue Type( questions, new requirements, bugs) Bug
Setup 2
• Hardware Platform (GPU) NVIDIA RTX 6000 Ada
• DeepStream Version 7.0
• TensorRT Version 8.6.1.6
• NVIDIA GPU Driver Version (valid for GPU only) 535.230.02
• Issue Type( questions, new requirements, bugs) Bug
Hello,
when running the simple python-pipeline provided below on both the systems noted above, a constant rise in RAM usage can be observed. While for this simple pipeline with the short input video with a length of 48 seconds, the rise is limited to around 2-4MB, in our much more complex production pipeline with 10+ input sources and three different models being infered we noticed the RAM usage slowly going up to the installed RAM of 64GB over a few days and thus crashing the application. We observed that behaviour for both file and RTSP input sources. We also observed a higher rising rate for models with more output values.
We’d like to kindly ask what the cause of this rising RAM usage is and how to prevent / limit it.
We hope that a possible solution for this simple pipeline can be transferred to our production pipeline.
Thank you very much for your help!
Simple pipeline to reproduce rising RAM usage.
import sys
import gi
gi.require_version("Gst", "1.0")
from gi.repository import GLib, Gst
# taken from deepstream-test3
def cb_newpad(decodebin, decoder_src_pad,data):
print("In cb_newpad\n")
caps=decoder_src_pad.get_current_caps()
if not caps:
caps = decoder_src_pad.query_caps()
gststruct=caps.get_structure(0)
gstname=gststruct.get_name()
source_bin=data
features=caps.get_features(0)
print("gstname=",gstname)
if(gstname.find("video")!=-1):
print("features=",features)
if features.contains("memory:NVMM"):
bin_ghost_pad=source_bin.get_static_pad("src")
if not bin_ghost_pad.set_target(decoder_src_pad):
sys.stderr.write("Failed to link decoder src pad to source bin ghost pad\n")
else:
sys.stderr.write(" Error: Decodebin did not pick nvidia decoder plugin.\n")
# taken from deepstream-test3
def decodebin_child_added(child_proxy,Object,name,user_data):
print("Decodebin child added:", name, "\n")
if(name.find("decodebin") != -1):
Object.connect("child-added",decodebin_child_added,user_data)
if "source" in name:
source_element = child_proxy.get_by_name("source")
if source_element.find_property("drop-on-latency") != None:
Object.set_property("drop-on-latency", True)
# taken from deepstream-test3
def create_source_bin(index,uri):
print("Creating source bin")
bin_name="source-bin-%02d" %index
print(bin_name)
nbin=Gst.Bin.new(bin_name)
if not nbin:
sys.stderr.write(" Unable to create source bin \n")
uri_decode_bin=Gst.ElementFactory.make("uridecodebin", "uri-decode-bin")
if not uri_decode_bin:
sys.stderr.write(" Unable to create uri decode bin \n")
uri_decode_bin.set_property("uri",uri)
uri_decode_bin.connect("pad-added",cb_newpad,nbin)
uri_decode_bin.connect("child-added",decodebin_child_added,nbin)
Gst.Bin.add(nbin,uri_decode_bin)
bin_pad=nbin.add_pad(Gst.GhostPad.new_no_target("src",Gst.PadDirection.SRC))
if not bin_pad:
sys.stderr.write(" Failed to add ghost pad in source bin \n")
return None
return nbin
if __name__ == "__main__":
Gst.init(None)
pipeline = Gst.Pipeline()
source_0_bin = create_source_bin(0, "file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4")
source_1_bin = create_source_bin(1, "file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4")
streammux = Gst.ElementFactory.make("nvstreammux", "streammux")
streammux.set_property("batch-size", 2)
pgie = Gst.ElementFactory.make("nvinfer", "pgie")
pgie.set_property("config-file-path", "/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.txt")
fakesink = Gst.ElementFactory.make("fakesink", "fakesink")
fakesink.set_property("sync", 1)
pipeline.add(source_0_bin)
pipeline.add(source_1_bin)
pipeline.add(streammux)
pipeline.add(pgie)
pipeline.add(fakesink)
source_0_bin_src_pad = source_0_bin.get_static_pad("src")
source_1_bin_src_pad = source_1_bin.get_static_pad("src")
streammux_sink_pad_0 = streammux.request_pad_simple("sink_0")
streammux_sink_pad_1 = streammux.request_pad_simple("sink_1")
source_0_bin_src_pad.link(streammux_sink_pad_0)
source_1_bin_src_pad.link(streammux_sink_pad_1)
streammux.link(pgie)
pgie.link(fakesink)
loop = GLib.MainLoop()
pipeline.set_state(Gst.State.PLAYING)
try:
loop.run()
except Exception as e:
print(e)
pipeline.set_state(Gst.State.NULL)
RAM usage of upper simple pipeline measured using psrecord. When using longer files, the rise continues.
# Elapsed time CPU (%) Real (MB) Virtual (MB)
0.000 0.000 471.254 56038.227
5.005 22.200 471.719 56102.227
10.011 29.600 472.074 56102.227
15.016 29.600 472.289 56102.227
20.021 28.200 472.504 56102.227
25.025 27.800 472.996 56102.227
30.030 27.400 473.219 56102.227
35.035 27.400 473.441 56102.227
40.040 23.400 473.691 56102.227
--> rise of 2.437MB RAM usage over 40 seconds.