Regarding the frame delay measurement method

Please provide complete information as applicable to your setup.

**• Hardware Platform (Jetson / GPU)Jetson
**• DeepStream Version6.3
**• JetPack Version (valid for Jetson only)5.1.2
• TensorRT Version
• NVIDIA GPU Driver Version (valid for GPU only)
**• Issue Type( questions, new requirements, bugs)question
• How to reproduce the issue ? (This is for bugs. Including which sample app is using, the configuration files content, the command line used and other details for reproducing)
• Requirement details( This is for new requirement. Including the module name-for which plugin or for which sample application, the function description)

Hi,
I would like to know how to measure the latency of each plugin when the deepstream application is running slow.

https://forums.developer.nvidia.com/t/latency-measure-for-every-frame/278047/6
I did the following with reference to

  1. pip install cffi
  2. apply the following patch to deepstream_test_3.py
diff --git a/apps/deepstream-test3/deepstream_test_3.py b/apps/deepstream-test3/deepstream_test_3.py
index d81ec92..21d2f3b 100755
--- a/apps/deepstream-test3/deepstream_test_3.py
+++ b/apps/deepstream-test3/deepstream_test_3.py
@@ -36,6 +36,28 @@ from common.FPS import PERF_DATA
 
 import pyds
 
+from cffi import FFI
+
+ffi = FFI()
+
+clib = None
+
+ffi.cdef("""
+typedef struct
+{
+  uint32_t source_id;
+  uint32_t frame_num;
+  double comp_in_timestamp;
+  double latency;
+} NvDsFrameLatencyInfo;
+
+uint32_t nvds_measure_buffer_latency(void *buf, NvDsFrameLatencyInfo *latency_info);
+bool nvds_get_enable_latency_measurement();
+""")
+
+# Compile the C sources to produce the following .dll (or .so under *nix)
+clib = ffi.dlopen("/opt/nvidia/deepstream/deepstream/lib/libnvdsgst_meta.so")
+
 no_display = False
 silent = False
 file_loop = False
@@ -56,6 +78,27 @@ OSD_PROCESS_MODE= 0
 OSD_DISPLAY_TEXT= 1
 pgie_classes_str= ["Vehicle", "TwoWheeler", "Person","RoadSign"]
 
+batch_num = 0
+
+def osd_src_pad_buffer_probe(pad, info, u_data):
+    number_source = u_data
+    gst_buffer = info.get_buffer()
+    if not gst_buffer:
+        print("Unable to get GstBuffer ")
+        return
+    global batch_num
+    if clib.nvds_get_enable_latency_measurement:
+        print(f"************BATCH-NUM = {batch_num}**************")
+        c_gst_buf = ffi.cast("void *", hash(gst_buffer))
+        cNvDsFrameLatencyInfo = ffi.new(f"NvDsFrameLatencyInfo[{number_source}]")
+        sources = clib.nvds_measure_buffer_latency(c_gst_buf, cNvDsFrameLatencyInfo)
+        for i in range(sources):
+            print(f"Source id = {cNvDsFrameLatencyInfo[i].source_id} "
+                  f"Frame_num = {cNvDsFrameLatencyInfo[i].frame_num} "
+                  f"Frame latency = {cNvDsFrameLatencyInfo[i].latency} (ms) ")
+        batch_num += 1
+    return Gst.PadProbeReturn.OK
+
 # pgie_src_pad_buffer_probe  will extract metadata received on tiler sink pad
 # and update params for drawing rectangle, object information etc.
 def pgie_src_pad_buffer_probe(pad,info,u_data):
@@ -199,7 +242,7 @@ def create_source_bin(index,uri):
         return None
     return nbin
 
-def main(args, requested_pgie=None, config=None, disable_probe=False):
+def main(args, requested_pgie=None, config=None, disable_probe=True):
     global perf_data
     perf_data = PERF_DATA(len(args))
 
@@ -380,6 +423,12 @@ def main(args, requested_pgie=None, config=None, disable_probe=False):
             # perf callback function to print fps every 5 sec
             GLib.timeout_add(5000, perf_data.perf_print_callback)
 
+    osd_src_pad=nvosd.get_static_pad("src")
+    if not osd_src_pad:
+        sys.stderr.write(" Unable to get src pad \n")
+    else:
+        osd_src_pad.add_probe(Gst.PadProbeType.BUFFER, osd_src_pad_buffer_probe, number_sources)
+
     # List the sources
     print("Now playing...")
     for i, source in enumerate(args):
  1. Execute the following command in the shell
export NVDS_ENABLE_COMPONENT_LATENCY_MEASUREMENT=1
export NVDS_ENABLE_LATENCY_MEASUREMENT=1

python3 deepstream_test_3.py --no-display -i rtsp://"yourtspuri0" uri1

When I run it, I get the following error

| Traceback (most recent call last):
| 
|     cNvDsFrameLatencyInfo = ffi.new(f"NvDsFrameLatencyInfo[{number_source}]")
|   File "/usr/local/lib/python3.8/dist-packages/cffi/api.py", line 266, in new
|     cdecl = self._typeof(cdecl)
|   File "/usr/local/lib/python3.8/dist-packages/cffi/api.py", line 186, in _typeof
|     result = self._typeof_locked(cdecl)
|   File "/usr/local/lib/python3.8/dist-packages/cffi/api.py", line 171, in _typeof_locked
|     type = self._parser.parse_type(cdecl)
|   File "/usr/local/lib/python3.8/dist-packages/cffi/cparser.py", line 552, in parse_type
|     return self.parse_type_and_quals(cdecl)[0]
|   File "/usr/local/lib/python3.8/dist-packages/cffi/cparser.py", line 555, in parse_type_and_quals
|     ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2]
|   File "/usr/local/lib/python3.8/dist-packages/cffi/cparser.py", line 338, in _parse
|     self.convert_pycparser_error(e, csource)
|   File "/usr/local/lib/python3.8/dist-packages/cffi/cparser.py", line 367, in convert_pycparser_error
|     raise CDefError(msg)
| cffi.CDefError: cannot parse "NvDsFrameLatencyInfo[<__gi__.GstNv3dSink object at 0xffff8363d480 (GstNv3dSink at 0x2406b750)>]"
| <cdef source string>:2:22: before: <
| ************BATCH-NUM = 0**************
| Traceback (most recent call last):
|   File "/usr/local/lib/python3.8/dist-packages/cffi/api.py", line 183, in _typeof
|     result = self._parsed_types[cdecl]
| KeyError: 'NvDsFrameLatencyInfo[<__gi__.GstNv3dSink object at 0xffff8363d480 (GstNv3dSink at 0x2406b750)>]'
| 
| During handling of the above exception, another exception occurred:
| 
| Traceback (most recent call last):
|   File "/usr/local/lib/python3.8/dist-packages/cffi/cparser.py", line 336, in _parse
| 
|     ast = _get_parser().parse(fullcsource)
|   File "/usr/local/lib/python3.8/dist-packages/pycparser/c_parser.py", line 147, in parse
|     return self.cparser.parse(
|   File "/usr/local/lib/python3.8/dist-packages/pycparser/ply/yacc.py", line 331, in parse
|     return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc)
|   File "/usr/local/lib/python3.8/dist-packages/pycparser/ply/yacc.py", line 1199, in parseopt_notrack
|     tok = call_errorfunc(self.errorfunc, errtoken, self)
|   File "/usr/local/lib/python3.8/dist-packages/pycparser/ply/yacc.py", line 193, in call_errorfunc
|     r = errorfunc(token)
|   File "/usr/local/lib/python3.8/dist-packages/pycparser/c_parser.py", line 1931, in p_error
|     self._parse_error(
|   File "/usr/local/lib/python3.8/dist-packages/pycparser/plyparser.py", line 67, in _parse_error
|     raise ParseError("%s: %s" % (coord, msg))
| pycparser.plyparser.ParseError: <cdef source string>:2:22: before: <
| 
| During handling of the above exception, another exception occurred:

What am I missing?

I am checking, the get back to you.

I am not able to reproduce this issue on Orin with DS7.0. here is the whole loglog-0709.txt (419.6 KB). here is the code deepstream_test_3.py (19.9 KB). is there any difference with my code? why do you add uri1 in the command-line?

Thank you for your thoughtful response.
I referred to the code you gave me and found some omissions on my part.

def pgie_src_pad_buffer_probe(pad,info,u_data):
+    global measure_latency
+    if measure_latency:
+        num_sources_in_batch = +pyds.nvds_measure_buffer_latency(hash(gst_buffer))
+        if num_sources_in_batch == 0:
+            print("Unable to get number of sources in GstBuffer for latency measurement")

def main(args, requested_pgie=None, config=None, disable_probe=True):
+    if environ.get('NVDS_ENABLE_LATENCY_MEASUREMENT') == '1':
+       print ("Pipeline Latency Measurement enabled!\nPlease set env var NVDS_ENABLE_COMPONENT_LATENCY_MEASUREMENT=1 for Component Latency Measurement")
+        global measure_latency
+        measure_latency = True

I added the above and ran it, but got the following error
module ‘pyds’ has no attribute ‘nvds_measure_buffer_latency’
I am currently running in the Dockerfile environment shown below.

FROM nvcr.io/nvidia/l4t-ml:r35.2.1-py3

# pip3 upgrade
RUN python3 -m pip install --upgrade pip

# DeepStream SDKインストール
RUN apt-get update && apt-get install -y \
#libssl3 \
libssl1.1 \
libssl-dev \
libgstreamer1.0-0 \
gstreamer1.0-tools \
gstreamer1.0-plugins-good \
gstreamer1.0-plugins-bad \
gstreamer1.0-plugins-ugly \
gstreamer1.0-libav \
libgstreamer-plugins-base1.0-dev \
libgstrtspserver-1.0-0 \
libjansson4 \
libyaml-cpp-dev
WORKDIR /tmp
RUN wget -q --content-disposition 'https://api.ngc.nvidia.com/v2/resources/org/nvidia/deepstream/6.3/files?redirect=true&path=deepstream_sdk_v6.3.0_jetson.tbz2' -O deepstream_sdk_v6.3.0_jetson.tbz2
RUN tar jxf deepstream_sdk_v6.3.0_jetson.tbz2 -C /
WORKDIR /opt/nvidia/deepstream/deepstream-6.3
RUN ./install.sh
RUN ldconfig

# DeepStream Python Apps Bindingsインストール
RUN apt-get update && apt-get install -y libcairo2-dev
RUN apt-get update && apt-get install -y libgirepository1.0-dev
RUN apt-get update && apt-get install -y libpython3.8-dev
WORKDIR /tmp
RUN wget -q https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/releases/download/v1.1.8/pyds-1.1.8-py3-none-linux_aarch64.whl
RUN pip3 install pyds-1.1.8-py3-none-linux_aarch64.whl

I am using pyds version “1.1.8”.
I checked if there is an attribute named nvds_measure_buffer_latency by print(dir(pyds)), and it is not there as shown in the error.
Is there any way to get it?

  1. about the the method in faq, except nvds_measure_buffer_latency ,are the other code the same with my code?
  2. nvds_measure_buffer_latency is a new method to print latency. it is added in version 1.1.11. deepstream_python_apps is opensource. you can port the code to 1.1.8. especially you need to rebuild the code and reinstall pyds after modifying the code. please refer to doc .

A1: Yes. It is the same.
A2: Okay. I think there is still a problem somewhere, so I will try to look at the doc you gave me.
Thank you very much.