Hey folks,
We have our python deepstream wrapper code sourced from Nvidia’s reference Python apps having our Trained unet Etlt model. I am trying to run this inside a Triton-docker container but it gets paused while processing on video clips after few frames. Where else code works fine for one frame /image. My same version of code works fine on Host machine for video clips as well as images without getting paused. I am running same python version and same dependency stack on both host machine as well inside container. also it not not throwing up any error message it just gets paused.
Any help and suggestion would be highly appreciated.
I am enclosing my Python Code and Config_file. along with the warning messages.
PythonWrapper:
#!/usr/bin/env python3
import sys
sys.path.append('../')
import gi
import math
gi.require_version('Gst', '1.0')
from gi.repository import GLib, Gst
from common.is_aarch_64 import is_aarch64
from common.bus_call import bus_call
import cv2
import pyds
import numpy as np
import os.path
from os import path
#import ctypes
#ctypes.pythonapi.PyCapsule_GetPointer.restype = ctypes.c_void_p
#ctypes.pythonapi.PyCapsule_GetPointer.argtypes = [ctypes.py_object, ctypes.c_char_p]
MAX_DISPLAY_LEN = 64
MUXER_OUTPUT_WIDTH = 1920
MUXER_OUTPUT_HEIGHT = 1080
MUXER_BATCH_TIMEOUT_USEC = 40#4000000
TILED_OUTPUT_WIDTH = 512
TILED_OUTPUT_HEIGHT = 512
COLORS = [[128, 128, 64], [0, 0, 128], [0, 128, 128], [128, 0, 0],
[128, 0, 128], [128, 128, 0], [0, 128, 0], [0, 0, 64],
[0, 0, 192], [0, 128, 64], [0, 128, 192], [128, 0, 64],
[128, 0, 192], [128, 128, 128]]
def map_mask_as_display_bgr(mask):
""" Assigning multiple colors as image output using the information
contained in mask. (BGR is opencv standard.)
"""
# getting a list of available classes
m_list = list(set(mask.flatten()))
print('m_list',m_list)
shp = mask.shape
print(np.unique(mask))
bgr = np.zeros((shp[0], shp[1], 3))#,dtype=np.int32)
print(np.unique(bgr))
for idx in m_list:
print((idx),COLORS[idx])
bgr[mask == idx] = COLORS[idx]
#bgr[mask == idx] = idx
print(np.unique(bgr))
#print(bgr)
return bgr
def seg_src_pad_buffer_probe(pad, info, u_data):
gst_buffer = info.get_buffer()
print(gst_buffer)
if not gst_buffer:
print("Unable to get GstBuffer ")
return
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
l_frame = batch_meta.frame_meta_list
while l_frame is not None:
try:
frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
print(frame_meta)
except StopIteration:
break
frame_number = frame_meta.frame_num
l_user = frame_meta.frame_user_meta_list
while l_user is not None:
try:
seg_user_meta = pyds.NvDsUserMeta.cast(l_user.data)
except StopIteration:
break
####TensorOutput
meta_type = seg_user_meta.base_meta.meta_type
if meta_type == pyds.NVDSINFER_TENSOR_OUTPUT_META:
meta = pyds.NvDsInferTensorMeta.cast(seg_user_meta.user_meta_data)
####SegmentatioMeta
if seg_user_meta and seg_user_meta.base_meta.meta_type == \
pyds.NVDSINFER_SEGMENTATION_META:
try:
segmeta = pyds.NvDsInferSegmentationMeta.cast(seg_user_meta.user_meta_data)
print(seg_user_meta.user_meta_data)
print('class',segmeta.classes)
except StopIteration:
break
print('classout',segmeta.classes)
masks = pyds.get_segmentation_masks(segmeta)
print('before',np.unique(np.array(masks)))
print('mask_shape',masks.shape)
#np.save('masks.npy',masks+1)
mask_final=masks+1
#print(np.unique(mask_final.astype(np.uint8),mask_final.shape))
cv2.imwrite(folder_name + "/" + str(frame_number) + ".jpg", mask_final.astype(np.uint8)) ##For writing mask Output to image file dir
masks = np.array(masks, copy=True, order='C')
print('after',np.unique(masks))
print(masks.shape)
print(masks)
print('class',segmeta.classes)
try:
l_user = l_user.next
except StopIteration:
break
try:
l_frame = l_frame.next
except StopIteration:
break
return Gst.PadProbeReturn.OK
def main(args):
# Check input arguments
if len(args) != 4:
sys.stderr.write("usage: %s config_file <jpeg/mjpeg file> "
"<path to save seg images>\n" % args[0])
sys.exit(1)
global folder_name
folder_name = args[-1]
if path.exists(folder_name):
sys.stderr.write("The output folder %s already exists. "
"Please remove it first.\n" % folder_name)
sys.exit(1)
os.mkdir(folder_name)
config_file = args[1]
num_sources = len(args) - 3
# Standard GStreamer initialization
Gst.init(None)
# Create gstreamer elements
# Create Pipeline element that will form a connection of other elements
print("Creating Pipeline \n ")
pipeline = Gst.Pipeline()
if not pipeline:
sys.stderr.write(" Unable to create Pipeline \n")
# Source element for reading from the file
print("Creating Source \n ")
source = Gst.ElementFactory.make("filesrc", "file-source")
if not source:
sys.stderr.write(" Unable to create Source \n")
# Since the data format in the input file is jpeg,
# we need a jpegparser
print("Creating jpegParser \n")
jpegparser = Gst.ElementFactory.make("jpegparse", "jpeg-parser")
if not jpegparser:
sys.stderr.write("Unable to create jpegparser \n")
# Use nvdec for hardware accelerated decode on GPU
print("Creating Decoder \n")
decoder = Gst.ElementFactory.make("nvv4l2decoder", "nvv4l2-decoder")
if not decoder:
sys.stderr.write(" Unable to create Nvv4l2 Decoder \n")
# Create nvstreammux instance to form batches from one or more sources.
streammux = Gst.ElementFactory.make("nvstreammux", "Stream-muxer")
if not streammux:
sys.stderr.write(" Unable to create NvStreamMux \n")
# Create segmentation for primary inference
seg = Gst.ElementFactory.make("nvinferbin", "primary-nvinference-engine")
if not seg:
sys.stderr.write("Unable to create primary inferene\n")
# Create nvsegvisual for visualizing segmentation
nvsegvisual = Gst.ElementFactory.make("nvsegvisual", "nvsegvisual")
if not nvsegvisual:
sys.stderr.write("Unable to create nvsegvisual\n")
if is_aarch64():
transform = Gst.ElementFactory.make("nvegltransform", "nvegl-transform")
print("Creating EGLSink \n")
#sink = Gst.ElementFactory.make("nveglglessink", "nvvideo-renderer")
sink = Gst.ElementFactory.make("filesink", "nvvideo-renderer")
if not sink:
sys.stderr.write(" Unable to create egl sink \n")
print("Playing file %s " % args[2])
source.set_property('location', args[2])
if is_aarch64() and (args[2].endswith("mjpeg") or args[2].endswith("mjpg")):
decoder.set_property('mjpeg', 1)
streammux.set_property('width', 1920)
streammux.set_property('height', 1080)
streammux.set_property('batch-size', 1)
streammux.set_property('batched-push-timeout', 4000000)
seg.set_property('config-file-path', config_file)
pgie_batch_size = seg.get_property("batch-size")
if pgie_batch_size != num_sources:
print("WARNING: Overriding infer-config batch-size", pgie_batch_size,
" with number of sources ", num_sources,
" \n")
seg.set_property("batch-size", num_sources)
nvsegvisual.set_property('batch-size', num_sources)
nvsegvisual.set_property('width', 512)
nvsegvisual.set_property('height', 512)
sink.set_property("qos", 0)
sink.set_property("location", 'sample_out.mkv')
print("Adding elements to Pipeline \n")
pipeline.add(source)
pipeline.add(jpegparser)
pipeline.add(decoder)
pipeline.add(streammux)
pipeline.add(seg)
pipeline.add(nvsegvisual)
pipeline.add(sink)
if is_aarch64():
pipeline.add(transform)
# we link the elements together
# file-source -> jpeg-parser -> nvv4l2-decoder ->
# nvinfer -> nvsegvisual -> sink
print("Linking elements in the Pipeline \n")
source.link(jpegparser)
jpegparser.link(decoder)
print('debug')
sinkpad = streammux.get_request_pad("sink_0")
if not sinkpad:
sys.stderr.write(" Unable to get the sink pad of streammux \n")
srcpad = decoder.get_static_pad("src")
if not srcpad:
sys.stderr.write(" Unable to get source pad of decoder \n")
srcpad.link(sinkpad)
streammux.link(seg)
seg.link(nvsegvisual)
if is_aarch64():
nvsegvisual.link(transform)
transform.link(sink)
else:
nvsegvisual.link(sink)
# create an event loop and feed gstreamer bus mesages to it
loop = GLib.MainLoop()
bus = pipeline.get_bus()
bus.add_signal_watch()
bus.connect("message", bus_call, loop)
# Lets add probe to get informed of the meta data generated, we add probe to
# the src pad of the inference element
seg_src_pad = seg.get_static_pad("src")
if not seg_src_pad:
sys.stderr.write(" Unable to get src pad \n")
else:
seg_src_pad.add_probe(Gst.PadProbeType.BUFFER, seg_src_pad_buffer_probe, 0)
# List the sources
print("Now playing...")
for i, source in enumerate(args[1:-1]):
if i != 0:
print(i, ": ", source)
print("Starting pipeline \n")
# start play back and listed to events
pipeline.set_state(Gst.State.PLAYING)
try:
loop.run()
print('#####')
except:
pass
# cleanup
pipeline.set_state(Gst.State.NULL)
if __name__ == '__main__':
sys.exit(main(sys.argv))
Config_File:
[property]
gpu-id=0
net-scale-factor=0.007843
# Since the model input channel is 3, using RGB color format.
model-color-format=1
offsets=127.5;127.5;127.5
workspace-size=10000
labelfile-path=../Model/labels.txt
tlt-encoded-model=model.etlt
tlt-model-key=model_key
infer-dims=3;512;512
batch-size=1
## 0=FP32, 1=INT8, 2=FP16 mode
network-mode=0
num-detected-classes=3
interval=0
gie-unique-id=1
network-type=2
output-blob-names=argmax_1
segmentation-threshold=0.0
maintain-aspect-ratio=0
segmentation-output-order=1
#output-tensor-meta=1
[class-attrs-all]
threshold=0.0
roi-top-offset=0
roi-bottom-offset=0
detected-min-w=0
detected-min-h=0
detected-max-w=0
detected-max-h=0
## Per class configuration
[class-attrs-2]
threshold=0.0
roi-top-offset=0
roi-bottom-offset=0
detected-min-w=0
detected-min-h=0
detected-max-w=0
detected-max-h=0
[class-attrs-3]
threshold=0.0
roi-top-offset=0
roi-bottom-offset=0
detected-min-w=0
detected-min-h=0
detected-max-w=0
detected-max-h=0
Std warnings inside Container:
(gst-plugin-scanner:33): GLib-GObject-WARNING **: 17:27:46.323: specified class size for type 'GstCompositor' is smaller than the parent type's 'GstVideoAggregator' class size
(gst-plugin-scanner:33): GLib-GObject-CRITICAL **: 17:27:46.324: g_type_add_interface_static: assertion 'G_TYPE_IS_INSTANTIATABLE (instance_type)' failed
(gst-plugin-scanner:33): GLib-CRITICAL **: 17:27:46.324: g_once_init_leave: assertion 'result != 0' failed
(gst-plugin-scanner:33): GStreamer-CRITICAL **: 17:27:46.324: gst_element_register: assertion 'g_type_is_a (type, GST_TYPE_ELEMENT)' failed
(gst-plugin-scanner:33): GStreamer-WARNING **: 17:27:46.753: Failed to load plugin '/usr/lib/x86_64-linux-gnu/gstreamer-1.0/deepstream/libnvdsgst_udp.so': librivermax.so.0: cannot open shared object file: No such file or directory
** (python:29): CRITICAL **: 17:28:17.156: _masked_scan_uint32_peek: assertion '(guint64) offset + size <= reader->size - reader->byte' failed
Environment
Deepstream/TensorRT Version:
deepstream-app version 6.0.1
DeepStreamSDK 6.0.1
CUDA Driver Version: 11.4
CUDA Runtime Version: 11.4
TensorRT Version: 8.2
cuDNN Version: 8.3
libNVWarp360 Version: 2.0.1d3
gst-launch-1.0 version 1.20.3
GStreamer 1.20.3
python 3.6.9
pyds 1.1.0
GPU Type:
TeslaT4
Container:
nvcr.io/nvidia/deepstream:6.0-ea-21.06-triton