Bounding boxes in Appsink for frames without detections

Accelerated Computing Intelligent Video Analytics DeepStream SDK
1

Please provide complete information as applicable to your setup.

** Jetson
** DeepStream 6.2
** Issue

Hi!

I want to set the inference interval to 5 (inference only every 5th frame). When doing this and adjusting the parameter correctly for multi object tracking (e.g. minTrackingConfidenceDuringInactive), I also get tracked bboxes for the non-inferences frames.

However, I don’t get these boxes when I query them in appsink. I know there are multiple threads (e.g. this or this) discussing this, but I’m not sure what the latest state on that is.

Could you tell me how I can get bboxes also for the intermittent frames in appsink?

Thanks!

3

How to reproduce your problem? Theappsink works as expected.

The following patch is based on /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-appsrc-test

diff --git a/sources/apps/sample_apps/deepstream-appsrc-test/deepstream_appsrc_test_app.c b/sources/apps/sample_apps/deepstream-appsrc-test/deepstream_appsrc_test_app.c
index 1c441ed..4539ca6 100644
--- a/sources/apps/sample_apps/deepstream-appsrc-test/deepstream_appsrc_test_app.c
+++ b/sources/apps/sample_apps/deepstream-appsrc-test/deepstream_appsrc_test_app.c
@@ -83,6 +83,11 @@ new_sample (GstElement * sink, gpointer * data)
           l_obj = l_obj->next) {
         obj_meta = (NvDsObjectMeta *) (l_obj->data);
         if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) {
+          printf("left %.2f top %.2f width %.2f height %.2f \n",
+                 obj_meta->tracker_bbox_info.org_bbox_coords.left,
+                 obj_meta->tracker_bbox_info.org_bbox_coords.top,
+                 obj_meta->tracker_bbox_info.org_bbox_coords.width,
+                 obj_meta->tracker_bbox_info.org_bbox_coords.height);
           vehicle_count++;
           num_rects++;
         }
@@ -210,6 +215,132 @@ usage(const char *bin)
     "<format(I420, NV12, RGBA)>\n", bin);
 }
 
+/* Tracker config parsing */
+#define CHECK_ERROR(error) \
+    if (error) { \
+        g_printerr ("Error while parsing config file: %s\n", error->message); \
+        goto done; \
+    }
+
+#define CONFIG_GROUP_TRACKER "tracker"
+#define CONFIG_GROUP_TRACKER_WIDTH "tracker-width"
+#define CONFIG_GROUP_TRACKER_HEIGHT "tracker-height"
+#define CONFIG_GROUP_TRACKER_LL_CONFIG_FILE "ll-config-file"
+#define CONFIG_GROUP_TRACKER_LL_LIB_FILE "ll-lib-file"
+#define CONFIG_GPU_ID "gpu-id"
+
+#define TRACKER_CONFIG_FILE "dstest2_tracker_config.txt"
+#define MAX_TRACKING_ID_LEN 16
+
+static gchar *
+get_absolute_file_path (gchar *cfg_file_path, gchar *file_path)
+{
+  gchar abs_cfg_path[PATH_MAX + 1];
+  gchar *abs_file_path;
+  gchar *delim;
+
+  if (file_path && file_path[0] == '/') {
+    return file_path;
+  }
+
+  if (!realpath (cfg_file_path, abs_cfg_path)) {
+    g_free (file_path);
+    return NULL;
+  }
+
+  // Return absolute path of config file if file_path is NULL.
+  if (!file_path) {
+    abs_file_path = g_strdup (abs_cfg_path);
+    return abs_file_path;
+  }
+
+  delim = g_strrstr (abs_cfg_path, "/");
+  *(delim + 1) = '\0';
+
+  abs_file_path = g_strconcat (abs_cfg_path, file_path, NULL);
+  g_free (file_path);
+
+  return abs_file_path;
+}
+
+static gboolean
+set_tracker_properties (GstElement *nvtracker)
+{
+  gboolean ret = FALSE;
+  GError *error = NULL;
+  gchar **keys = NULL;
+  gchar **key = NULL;
+  GKeyFile *key_file = g_key_file_new ();
+
+  if (!g_key_file_load_from_file (key_file, TRACKER_CONFIG_FILE, G_KEY_FILE_NONE,
+          &error)) {
+    if (error) {
+      g_printerr ("Failed to load config file: %s\n", error->message);
+      g_error_free (error);
+    } else {
+      g_printerr ("Failed to load config file.\n");
+    }
+    return FALSE;
+  }
+
+  keys = g_key_file_get_keys (key_file, CONFIG_GROUP_TRACKER, NULL, &error);
+  CHECK_ERROR (error);
+
+  for (key = keys; *key; key++) {
+    if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_WIDTH)) {
+      gint width =
+          g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER,
+          CONFIG_GROUP_TRACKER_WIDTH, &error);
+      CHECK_ERROR (error);
+      g_object_set (G_OBJECT (nvtracker), "tracker-width", width, NULL);
+    } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_HEIGHT)) {
+      gint height =
+          g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER,
+          CONFIG_GROUP_TRACKER_HEIGHT, &error);
+      CHECK_ERROR (error);
+      g_object_set (G_OBJECT (nvtracker), "tracker-height", height, NULL);
+    } else if (!g_strcmp0 (*key, CONFIG_GPU_ID)) {
+      guint gpu_id =
+          g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER,
+          CONFIG_GPU_ID, &error);
+      CHECK_ERROR (error);
+      g_object_set (G_OBJECT (nvtracker), "gpu_id", gpu_id, NULL);
+    } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_LL_CONFIG_FILE)) {
+      char* ll_config_file = get_absolute_file_path (TRACKER_CONFIG_FILE,
+                g_key_file_get_string (key_file,
+                    CONFIG_GROUP_TRACKER,
+                    CONFIG_GROUP_TRACKER_LL_CONFIG_FILE, &error));
+      CHECK_ERROR (error);
+      g_object_set (G_OBJECT (nvtracker), "ll-config-file", ll_config_file, NULL);
+      g_free(ll_config_file);
+    } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_LL_LIB_FILE)) {
+      char* ll_lib_file = get_absolute_file_path (TRACKER_CONFIG_FILE,
+                g_key_file_get_string (key_file,
+                    CONFIG_GROUP_TRACKER,
+                    CONFIG_GROUP_TRACKER_LL_LIB_FILE, &error));
+      CHECK_ERROR (error);
+      g_object_set (G_OBJECT (nvtracker), "ll-lib-file", ll_lib_file, NULL);
+      g_free(ll_lib_file);
+    } else {
+      g_printerr ("Unknown key '%s' for group [%s]", *key,
+          CONFIG_GROUP_TRACKER);
+    }
+  }
+
+  ret = TRUE;
+done:
+  if (error) {
+    g_error_free (error);
+  }
+  if (keys) {
+    g_strfreev (keys);
+  }
+  if (!ret) {
+    g_printerr ("%s failed", __func__);
+  }
+  return ret;
+}
+
 int
 main (int argc, char *argv[])
 {
@@ -354,6 +485,14 @@ main (int argc, char *argv[])
     return -1;
   }
 
+  /* We need to have a tracker to track the identified objects */
+  GstElement *nvtracker = gst_element_factory_make ("nvtracker", "tracker");
+  /* Set necessary properties of the tracker element. */
+  if (!set_tracker_properties(nvtracker)) {
+    g_printerr ("Failed to set tracker properties. Exiting.\n");
+    return -1;
+  }
+
   /* Use convertor to convert from NV12 to RGBA as required by nvdsosd */
   nvvidconv2 =
       gst_element_factory_make ("nvvideoconvert", "nvvideo-converter2");
@@ -445,7 +584,7 @@ main (int argc, char *argv[])
   /* Set up the pipeline */
   /* we add all elements into the pipeline */
   gst_bin_add_many (GST_BIN (pipeline),
-      data.app_source, nvvidconv1, caps_filter, streammux, pgie,
+      data.app_source, nvvidconv1, caps_filter, streammux, pgie, nvtracker,
       nvvidconv2, nvosd, tee, sink, appsink, NULL);
 
   GstPad *sinkpad, *srcpad;
@@ -478,12 +617,12 @@ main (int argc, char *argv[])
 
   if (!gst_element_link_many (data.app_source, nvvidconv1, caps_filter, NULL) ||
       !gst_element_link_many (nvosd, sink, NULL) ||
-      !gst_element_link_many (streammux, pgie, nvvidconv2, tee, NULL)) {
+      !gst_element_link_many (streammux, pgie, nvtracker, nvvidconv2, tee, NULL)) {
     g_printerr ("Elements could not be linked: Exiting.\n");
     return -1;
   }
 
-/* Manually link the Tee, which has "Request" pads.
+  /* Manually link the Tee, which has "Request" pads.
  * This tee, in case of multistream usecase, will come before tiler element. */
   tee_source_pad1 = gst_element_request_pad_simple (tee, "src_0");
   osd_sink_pad = gst_element_get_static_pad (nvosd, "sink");
diff --git a/sources/apps/sample_apps/deepstream-appsrc-test/dstest2_tracker_config.txt b/sources/apps/sample_apps/deepstream-appsrc-test/dstest2_tracker_config.txt
new file mode 100644
index 0000000..7a315a5
--- /dev/null
+++ b/sources/apps/sample_apps/deepstream-appsrc-test/dstest2_tracker_config.txt
@@ -0,0 +1,29 @@
+################################################################################
+# SPDX-FileCopyrightText: Copyright (c) 2018-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: LicenseRef-NvidiaProprietary
+#
+# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
+# property and proprietary rights in and to this material, related
+# documentation and any modifications thereto. Any use, reproduction,
+# disclosure or distribution of this material and related documentation
+# without an express license agreement from NVIDIA CORPORATION or
+# its affiliates is strictly prohibited.
+################################################################################
+
+# Mandatory properties for the tracker:
+#   tracker-width, tracker-height: needs to be multiple of 32 for NvDCF and NvDeepSORT
+#   gpu-id
+#   ll-lib-file: path to low-level tracker lib
+#   ll-config-file: required to set different tracker types
+#
+[tracker]
+tracker-width=960
+tracker-height=544
+gpu-id=0
+ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so
+# ll-config-file required to set different tracker types
+# ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_IOU.yml
+# ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_NvSORT.yml
+ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml
+# ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml
+# ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_NvDeepSORT.yml
diff --git a/sources/apps/sample_apps/deepstream-appsrc-test/dstest_appsrc_config.txt b/sources/apps/sample_apps/deepstream-appsrc-test/dstest_appsrc_config.txt
index 2f73b0a..0c811bb 100644
--- a/sources/apps/sample_apps/deepstream-appsrc-test/dstest_appsrc_config.txt
+++ b/sources/apps/sample_apps/deepstream-appsrc-test/dstest_appsrc_config.txt
@@ -57,7 +57,7 @@ int8-calib-file=../../../../samples/models/Primary_Detector/cal_trt.bin
 batch-size=1
 network-mode=1
 num-detected-classes=4
-interval=0
+interval=5
 gie-unique-id=1
 #scaling-filter=0
 #scaling-compute-hw=0

There is a known issue here, nvtracker with appsink will get stuck in the last frame, this problem will be fixed in the next version

4

Thanks for your reply!

Indeed, even bboxes (obj_meta.rect_params) are provided for non-inference frames. I wasn’t seeing them first as I had filtered out boxes that have lower than 0 confidence. That brings me to a short follow-up question on which values to trust or the meaning of these values. Which confidence would I take for non-inferenced frames for the objects that keep getting tracked? It seems that the tracker confidence is much higher most of the time. E.g., see this:

Tracking BBox: left=223.79107666015625, top=94.87663269042969, width=1502.1065673828125,
Tracker confidence: 0.93
objects [{‘obj_id’: 0, ‘category’: ‘boat’, ‘confidence’: 0.18978506326675415, ‘x’: 223.79107666015625, ‘y’: 94.87663269042969, ‘width’: 1502.1065673828125, ‘height’: 887.6854248046875, ‘alertstate’: -1, ‘x_world’: -1, ‘y_world’: -1, ‘distance’: -1}]
Tracking BBox: left=225.477783203125, top=93.97451782226562, width=1502.1063232421875,
Tracker confidence: 0.92
objects [{‘obj_id’: 0, ‘category’: ‘boat’, ‘confidence’: -0.10000000149011612, ‘x’: 225.477783203125, ‘y’: 93.97451782226562, ‘width’: 1502.1063232421875, ‘height’: 887.685546875, ‘alertstate’: -1, ‘x_world’: -1, ‘y_world’: -1, ‘distance’: -1}]
Tracking BBox: left=225.60791015625, top=92.86370086669922, width=1502.1064453125,
Tracker confidence: 0.93
objects [{‘obj_id’: 0, ‘category’: ‘boat’, ‘confidence’: -0.10000000149011612, ‘x’: 225.60791015625, ‘y’: 92.86370086669922, ‘width’: 1502.1064453125, ‘height’: 887.685546875, ‘alertstate’: -1, ‘x_world’: -1, ‘y_world’: -1, ‘distance’: -1}]
Tracking BBox: left=231.216064453125, top=94.8365707397461, width=1499.5762939453125,
Tracker confidence: 0.93
objects [{‘obj_id’: 0, ‘category’: ‘boat’, ‘confidence’: 0.19587963819503784, ‘x’: 231.216064453125, ‘y’: 94.8365707397461, ‘width’: 1499.5762939453125, ‘height’: 886.5253295898438, ‘alertstate’: -1, ‘x_world’: -1, ‘y_world’: -1, ‘distance’: -1}]
Tracking BBox: left=231.9730682373047, top=94.67962646484375, width=1499.576416015625,
Tracker confidence: 0.94
objects [{‘obj_id’: 0, ‘category’: ‘boat’, ‘confidence’: -0.10000000149011612, ‘x’: 231.9730682373047, ‘y’: 94.67962646484375, ‘width’: 1499.576416015625, ‘height’: 886.5253295898438, ‘alertstate’: -1, ‘x_world’: -1, ‘y_world’: -1, ‘distance’: -1}]
Tracking BBox: left=231.0697021484375, top=93.61817932128906, width=1499.576416015625,
Tracker confidence: 0.96
objects [{‘obj_id’: 0, ‘category’: ‘boat’, ‘confidence’: -0.10000000149011612, ‘x’: 231.0697021484375, ‘y’: 93.61817932128906, ‘width’: 1499.576416015625, ‘height’: 886.5253295898438, ‘alertstate’: -1, ‘x_world’: -1, ‘y_world’: -1, ‘distance’: -1}]
nvstreammux: Successfully handled EOS for source_id=0
Tracking BBox: left=231.0496826171875, top=90.43321228027344, width=1500.0833740234375,
Tracker confidence: 0.96
objects [{‘obj_id’: 0, ‘category’: ‘boat’, ‘confidence’: 0.19709855318069458, ‘x’: 231.0496826171875, ‘y’: 90.43321228027344, ‘width’: 1500.0833740234375, ‘height’: 894.9386596679688, ‘alertstate’: -1, ‘x_world’: -1, ‘y_world’: -1, ‘distance’: -1}]

For non-inference frames the confidence is always -0.10000000149011612, for inference frames it’s pretty low in these examples (0.19709855318069458). At the same time, the tracker confidence is almost 1.

Which confidence should I take for non-inference frames? Ideally, it should be a mix of the OD confidence and the tracker confidence?

Thank you!

5

This is related to your configuration file
You can refer to the comments in sources/includes/nvdsmeta.h

  /** Holds a confidence value for the object, set by the inference
   component. confidence will be set to -0.1, if "Group Rectangles" mode of
   clustering is chosen since the algorithm does not preserve confidence
   values. Also, for objects found by tracker and not inference component,
   confidence will be set to -0.1 */
  gfloat confidence;
  /** Holds a confidence value for the object set by nvdcf_tracker.
   * tracker_confidence will be set to -0.1 for KLT and IOU tracker */
  gfloat tracker_confidence;
6

OK thanks, that means, I have to work with -0.1 and the tracker confidence as there is no other “fused” confidence, right?

I was thinking of a simple “nearest neighbour” confidence that just passes the confidence over. A bit more complicated would be a weighted confidence that takes the previous inference confidence and multiplies it by the tracker confidence.

7

I think so.

Perhaps you can use the following member as a workaround

  /** Holds additional user-defined object information. */
  gint64 misc_obj_info[MAX_USER_FIELDS];
  /** For internal use. */
  gint64 reserved[MAX_RESERVED_FIELDS];
}NvDsObjectMeta;