Deepstream pauses on processing onto video frames inside triton container after few frames

Hey folks,

We have our python deepstream wrapper code sourced from Nvidia’s reference Python apps having our Trained unet Etlt model. I am trying to run this inside a Triton-docker container but it gets paused while processing on video clips after few frames. Where else code works fine for one frame /image. My same version of code works fine on Host machine for video clips as well as images without getting paused. I am running same python version and same dependency stack on both host machine as well inside container. also it not not throwing up any error message it just gets paused.

Any help and suggestion would be highly appreciated.

I am enclosing my Python Code and Config_file. along with the warning messages.

PythonWrapper:

#!/usr/bin/env python3

import sys

sys.path.append('../')
import gi
import math

gi.require_version('Gst', '1.0')
from gi.repository import GLib, Gst
from common.is_aarch_64 import is_aarch64
from common.bus_call import bus_call

import cv2
import pyds
import numpy as np
import os.path
from os import path

#import ctypes

#ctypes.pythonapi.PyCapsule_GetPointer.restype = ctypes.c_void_p
#ctypes.pythonapi.PyCapsule_GetPointer.argtypes = [ctypes.py_object, ctypes.c_char_p]




MAX_DISPLAY_LEN = 64
MUXER_OUTPUT_WIDTH = 1920
MUXER_OUTPUT_HEIGHT = 1080
MUXER_BATCH_TIMEOUT_USEC = 40#4000000
TILED_OUTPUT_WIDTH = 512
TILED_OUTPUT_HEIGHT = 512
COLORS = [[128, 128, 64], [0, 0, 128], [0, 128, 128], [128, 0, 0],
          [128, 0, 128], [128, 128, 0], [0, 128, 0], [0, 0, 64],
          [0, 0, 192], [0, 128, 64], [0, 128, 192], [128, 0, 64],
          [128, 0, 192], [128, 128, 128]]




def map_mask_as_display_bgr(mask):
    """ Assigning multiple colors as image output using the information
        contained in mask. (BGR is opencv standard.)
    """
    # getting a list of available classes
    m_list = list(set(mask.flatten()))
    print('m_list',m_list)

    shp = mask.shape
    print(np.unique(mask))
    bgr = np.zeros((shp[0], shp[1], 3))#,dtype=np.int32)
    print(np.unique(bgr))
    for idx in m_list:
        print((idx),COLORS[idx])
        bgr[mask == idx] = COLORS[idx]
        #bgr[mask == idx] = idx
    print(np.unique(bgr))
    #print(bgr)
    return bgr


def seg_src_pad_buffer_probe(pad, info, u_data):
    gst_buffer = info.get_buffer()
    print(gst_buffer)
    if not gst_buffer:
        print("Unable to get GstBuffer ")
        return

    batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
    l_frame = batch_meta.frame_meta_list
    while l_frame is not None:
        try:
            frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
            print(frame_meta)
           

        except StopIteration:
            break
        frame_number = frame_meta.frame_num
        l_user = frame_meta.frame_user_meta_list
        while l_user is not None:
            try:
                seg_user_meta = pyds.NvDsUserMeta.cast(l_user.data)
            except StopIteration:
                break
            ####TensorOutput
            meta_type = seg_user_meta.base_meta.meta_type
            if meta_type == pyds.NVDSINFER_TENSOR_OUTPUT_META:
                meta = pyds.NvDsInferTensorMeta.cast(seg_user_meta.user_meta_data)




            
            ####SegmentatioMeta
            if seg_user_meta and seg_user_meta.base_meta.meta_type == \
                    pyds.NVDSINFER_SEGMENTATION_META:
                try:
                    segmeta = pyds.NvDsInferSegmentationMeta.cast(seg_user_meta.user_meta_data)
                    print(seg_user_meta.user_meta_data)
                    print('class',segmeta.classes)
                except StopIteration:
                    break

                print('classout',segmeta.classes)
                masks = pyds.get_segmentation_masks(segmeta)
                print('before',np.unique(np.array(masks)))
                print('mask_shape',masks.shape)
                #np.save('masks.npy',masks+1)
                mask_final=masks+1
                #print(np.unique(mask_final.astype(np.uint8),mask_final.shape))
                cv2.imwrite(folder_name + "/" + str(frame_number) + ".jpg", mask_final.astype(np.uint8)) ##For writing mask Output to image file dir
 
                masks = np.array(masks, copy=True, order='C')
                print('after',np.unique(masks))
                print(masks.shape)
                print(masks)
                
                print('class',segmeta.classes)                
            try:
                l_user = l_user.next
            except StopIteration:
                break
        try:
            l_frame = l_frame.next
        except StopIteration:
            break
    return Gst.PadProbeReturn.OK


def main(args):
    # Check input arguments
    if len(args) != 4:
        sys.stderr.write("usage: %s config_file <jpeg/mjpeg file> "
                         "<path to save seg images>\n" % args[0])
        sys.exit(1)

    global folder_name
    folder_name = args[-1]
    if path.exists(folder_name):
        sys.stderr.write("The output folder %s already exists. "
                         "Please remove it first.\n" % folder_name)
        sys.exit(1)
    os.mkdir(folder_name)

    config_file = args[1]
    num_sources = len(args) - 3
    # Standard GStreamer initialization
    Gst.init(None)

    # Create gstreamer elements
    # Create Pipeline element that will form a connection of other elements
    print("Creating Pipeline \n ")
    pipeline = Gst.Pipeline()

    if not pipeline:
        sys.stderr.write(" Unable to create Pipeline \n")

    # Source element for reading from the file
    print("Creating Source \n ")
    source = Gst.ElementFactory.make("filesrc", "file-source")
    if not source:
        sys.stderr.write(" Unable to create Source \n")

    # Since the data format in the input file is jpeg,
    # we need a jpegparser
    print("Creating jpegParser \n")
    jpegparser = Gst.ElementFactory.make("jpegparse", "jpeg-parser")
    if not jpegparser:
        sys.stderr.write("Unable to create jpegparser \n")

    # Use nvdec for hardware accelerated decode on GPU
    print("Creating Decoder \n")
    decoder = Gst.ElementFactory.make("nvv4l2decoder", "nvv4l2-decoder")
    if not decoder:
        sys.stderr.write(" Unable to create Nvv4l2 Decoder \n")

    # Create nvstreammux instance to form batches from one or more sources.
    streammux = Gst.ElementFactory.make("nvstreammux", "Stream-muxer")
    if not streammux:
        sys.stderr.write(" Unable to create NvStreamMux \n")

    # Create segmentation for primary inference
    seg = Gst.ElementFactory.make("nvinferbin", "primary-nvinference-engine")
    if not seg:
        sys.stderr.write("Unable to create primary inferene\n")

    # Create nvsegvisual for visualizing segmentation
    nvsegvisual = Gst.ElementFactory.make("nvsegvisual", "nvsegvisual")
    if not nvsegvisual:
        sys.stderr.write("Unable to create nvsegvisual\n")

    if is_aarch64():
        transform = Gst.ElementFactory.make("nvegltransform", "nvegl-transform")

    print("Creating EGLSink \n")
    #sink = Gst.ElementFactory.make("nveglglessink", "nvvideo-renderer")
    sink = Gst.ElementFactory.make("filesink", "nvvideo-renderer")
    if not sink:
        sys.stderr.write(" Unable to create egl sink \n")

    print("Playing file %s " % args[2])
    source.set_property('location', args[2])
    if is_aarch64() and (args[2].endswith("mjpeg") or args[2].endswith("mjpg")):
        decoder.set_property('mjpeg', 1)
    streammux.set_property('width', 1920)
    streammux.set_property('height', 1080)
    streammux.set_property('batch-size', 1)
    streammux.set_property('batched-push-timeout', 4000000)
    seg.set_property('config-file-path', config_file)
    pgie_batch_size = seg.get_property("batch-size")
    if pgie_batch_size != num_sources:
        print("WARNING: Overriding infer-config batch-size", pgie_batch_size,
              " with number of sources ", num_sources,
              " \n")
        seg.set_property("batch-size", num_sources)
    nvsegvisual.set_property('batch-size', num_sources)
    nvsegvisual.set_property('width', 512)
    nvsegvisual.set_property('height', 512)
    sink.set_property("qos", 0)
    sink.set_property("location", 'sample_out.mkv')
    print("Adding elements to Pipeline \n")
    pipeline.add(source)
    pipeline.add(jpegparser)
    pipeline.add(decoder)
    pipeline.add(streammux)
    pipeline.add(seg)
    pipeline.add(nvsegvisual)
    pipeline.add(sink)
    
    if is_aarch64():
        pipeline.add(transform)

    # we link the elements together
    # file-source -> jpeg-parser -> nvv4l2-decoder ->
    # nvinfer -> nvsegvisual -> sink
    print("Linking elements in the Pipeline \n")
    source.link(jpegparser)
    jpegparser.link(decoder)
    print('debug')
    sinkpad = streammux.get_request_pad("sink_0")
    if not sinkpad:
        sys.stderr.write(" Unable to get the sink pad of streammux \n")
    srcpad = decoder.get_static_pad("src")
    if not srcpad:
        sys.stderr.write(" Unable to get source pad of decoder \n")
    srcpad.link(sinkpad)
    streammux.link(seg)
    seg.link(nvsegvisual)
    if is_aarch64():
        nvsegvisual.link(transform)
        transform.link(sink)
    else:
        nvsegvisual.link(sink)
    # create an event loop and feed gstreamer bus mesages to it
    loop = GLib.MainLoop()
    bus = pipeline.get_bus()
    bus.add_signal_watch()
    bus.connect("message", bus_call, loop)

    # Lets add probe to get informed of the meta data generated, we add probe to
    # the src pad of the inference element
    seg_src_pad = seg.get_static_pad("src")
    if not seg_src_pad:
        sys.stderr.write(" Unable to get src pad \n")
    else:
        seg_src_pad.add_probe(Gst.PadProbeType.BUFFER, seg_src_pad_buffer_probe, 0)

    # List the sources
    print("Now playing...")
    for i, source in enumerate(args[1:-1]):
        if i != 0:
            print(i, ": ", source)

    print("Starting pipeline \n")
    # start play back and listed to events
    pipeline.set_state(Gst.State.PLAYING)
    try:
        loop.run()
        print('#####')
    except:
        pass
    # cleanup
    pipeline.set_state(Gst.State.NULL)


if __name__ == '__main__':
    sys.exit(main(sys.argv))

Config_File:

[property]
gpu-id=0
net-scale-factor=0.007843
# Since the model input channel is 3, using RGB color format.
model-color-format=1
offsets=127.5;127.5;127.5
workspace-size=10000


labelfile-path=../Model/labels.txt

tlt-encoded-model=model.etlt
tlt-model-key=model_key


infer-dims=3;512;512
batch-size=1
## 0=FP32, 1=INT8, 2=FP16 mode
network-mode=0
num-detected-classes=3
interval=0
gie-unique-id=1
network-type=2
output-blob-names=argmax_1
segmentation-threshold=0.0
maintain-aspect-ratio=0
segmentation-output-order=1
#output-tensor-meta=1


[class-attrs-all]
threshold=0.0
roi-top-offset=0
roi-bottom-offset=0
detected-min-w=0
detected-min-h=0
detected-max-w=0
detected-max-h=0

## Per class configuration
[class-attrs-2]
threshold=0.0
roi-top-offset=0
roi-bottom-offset=0
detected-min-w=0
detected-min-h=0
detected-max-w=0
detected-max-h=0

[class-attrs-3]
threshold=0.0
roi-top-offset=0
roi-bottom-offset=0
detected-min-w=0
detected-min-h=0
detected-max-w=0
detected-max-h=0

Std warnings inside Container:

(gst-plugin-scanner:33): GLib-GObject-WARNING **: 17:27:46.323: specified class size for type 'GstCompositor' is smaller than the parent type's 'GstVideoAggregator' class size

(gst-plugin-scanner:33): GLib-GObject-CRITICAL **: 17:27:46.324: g_type_add_interface_static: assertion 'G_TYPE_IS_INSTANTIATABLE (instance_type)' failed

(gst-plugin-scanner:33): GLib-CRITICAL **: 17:27:46.324: g_once_init_leave: assertion 'result != 0' failed

(gst-plugin-scanner:33): GStreamer-CRITICAL **: 17:27:46.324: gst_element_register: assertion 'g_type_is_a (type, GST_TYPE_ELEMENT)' failed

(gst-plugin-scanner:33): GStreamer-WARNING **: 17:27:46.753: Failed to load plugin '/usr/lib/x86_64-linux-gnu/gstreamer-1.0/deepstream/libnvdsgst_udp.so': librivermax.so.0: cannot open shared object file: No such file or directory

** (python:29): CRITICAL **: 17:28:17.156: _masked_scan_uint32_peek: assertion '(guint64) offset + size <= reader->size - reader->byte' failed

Environment

Deepstream/TensorRT Version:

deepstream-app version 6.0.1
DeepStreamSDK 6.0.1
CUDA Driver Version: 11.4
CUDA Runtime Version: 11.4
TensorRT Version: 8.2
cuDNN Version: 8.3
libNVWarp360 Version: 2.0.1d3
gst-launch-1.0 version 1.20.3
GStreamer 1.20.3
python 3.6.9
pyds 1.1.0

GPU Type:

TeslaT4

Container:

nvcr.io/nvidia/deepstream:6.0-ea-21.06-triton

could you share more logs? please do “export GST_DEBUG=5”, then run again, you can redirect the logs to a file.

0:00:00.304999291   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305036282   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2924:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe minimum capture size for pixelformat MJPG
0:00:00.305048737   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305059347   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2930:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe maximum capture size for pixelformat MJPG
0:00:00.305080237   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305090257   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2924:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe minimum capture size for pixelformat MPG4
0:00:00.305099504   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305109053   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2930:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe maximum capture size for pixelformat MPG4
0:00:00.305123871   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305133620   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2924:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe minimum capture size for pixelformat MPG2
0:00:00.305142377   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305150973   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2930:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe maximum capture size for pixelformat MPG2
0:00:00.305164399   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305171984   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2924:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe minimum capture size for pixelformat H265
0:00:00.305184407   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305190990   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2930:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe maximum capture size for pixelformat H265
0:00:00.305202903   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305213123   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2924:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe minimum capture size for pixelformat VP90
0:00:00.305220156   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305227300   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2930:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe maximum capture size for pixelformat VP90
0:00:00.305237320   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305243601   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2924:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe minimum capture size for pixelformat VP80
0:00:00.305249343   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305255444   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2930:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe maximum capture size for pixelformat VP80
0:00:00.305266476   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305274220   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2924:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe minimum capture size for pixelformat H264
0:00:00.305280402   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:sink> Unable to try format: Unknown error -1
0:00:00.305286594   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2930:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:sink> Could not probe maximum capture size for pixelformat H264
0:00:00.305683218   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:src> Unable to try format: Unknown error -1
0:00:00.305697545   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2924:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:src> Could not probe minimum capture size for pixelformat NM12
0:00:00.305705170   107      0x3343630 WARN                    v4l2 gstv4l2object.c:3038:gst_v4l2_object_get_nearest_size:<nvv4l2-decoder:src> Unable to try format: Unknown error -1
0:00:00.305714908   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2930:gst_v4l2_object_probe_caps_for_format:<nvv4l2-decoder:src> Could not probe maximum capture size for pixelformat NM12
0:00:00.305726591   107      0x3343630 WARN                    v4l2 gstv4l2object.c:2375:gst_v4l2_object_add_interlace_mode:0x2e88f70 Failed to determine interlace mode
0:00:00.305965551   107      0x3343630 INFO                 nvinfer gstnvinfer.cpp:619:gst_nvinfer_logger:<nvinfer_bin_nvinfer> NvDsInferContext[UID 1]: Info from NvDsInferContextImpl::buildModel() <nvdsinfer_context_impl.cpp:1818> [UID = 1]: Trying to create engine from model files

the log is too short, and no error log, did it crash at beginning or after a while?
could you share the whole logs? please do “export GST_DEBUG=5”, then run again, you can redirect the logs to a file.

DEBUG_LOG.txt (15.7 MB)

please use fakesink or use filesink with encoding, you can refer to deepstream_preprocess_test.py

I have been already using the FileSink you can also have look into the python Wrapper I have also provided in description. Similarly we tried using using Fakesink in past but since it gives standard output on Display and we are behind the VM we didn’t had the very good luck with it

Fixed the issue after using Fakesink and Latest Triton image.

Glad to know you fixed it, thanks for the update! If need further support, please open a new one. Thanks

1 Like