Bug of nvdsmultistreamtiler on DeepStream 6.1.1 Jetson

[v.hunglx2]

Please provide complete information as applicable to your setup.

• Hardware Platform (Jetson / GPU) Jetson
• DeepStream Version 6.1.1
• JetPack Version (valid for Jetson only) 5.0.2
• TensorRT Version 8.4
• Issue Type( questions, new requirements, bugs) bugs
On Jetson 5.0.2 and DeepStream 6.1.1, multistreamtiler plugin do not rescale NvOSD_ArrowParams of display metadata. That cause some arrow display position errors.

[yuweiw]

What pipeline did you run, our demo app or your own app?Could you please show us the code if it’s your own code?

[v.hunglx2]

Should be easy to reproduce following this sample code.

//
// Created by hunglx on 17/10/2022.
//

#include <gtest/gtest.h>
#include <gst/gst.h>
#include <nvdsmeta.h>
#include <gstnvdsmeta.h>

static GstPadProbeReturn
mux_src_probe(GstPad *pad, GstPadProbeInfo *info, gpointer user_data) {
  GstBuffer *gst_buffer = static_cast<GstBuffer *>(info->data);

  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(gst_buffer);
  for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != nullptr; l_frame = l_frame->next) {
    NvDsFrameMeta *frame_meta = static_cast<NvDsFrameMeta *>(l_frame->data);
    NvDsDisplayMeta *display_meta = nvds_acquire_display_meta_from_pool(batch_meta);
    display_meta->num_arrows = 0;

    NvOSD_ArrowParams &arrow = display_meta->arrow_params[display_meta->num_arrows];
    display_meta->num_arrows++;
    arrow.x1 = 200;
    arrow.y1 = 200;
    arrow.x2 = 300;
    arrow.y2 = 300;
    arrow.arrow_width = 4;
    arrow.arrow_head = END_HEAD;
    arrow.arrow_color = NvOSD_ColorParams{.red=0.0, .green=1.0, .blue=0.0, .alpha=1.0};

    nvds_add_display_meta_to_frame(frame_meta, display_meta);
  }

  return GST_PAD_PROBE_OK;
}

static GstPadProbeReturn
tiler_src_probe(GstPad *pad, GstPadProbeInfo *info, gpointer user_data) {
  GstBuffer *gst_buffer = static_cast<GstBuffer *>(info->data);

  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(gst_buffer);
  for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != nullptr; l_frame = l_frame->next) {
    NvDsFrameMeta *frame_meta = static_cast<NvDsFrameMeta *>(l_frame->data);

    for (NvDsMetaList *l_display = frame_meta->display_meta_list; l_display != nullptr; l_display = l_display->next) {
      NvDsDisplayMeta *display_meta = static_cast<NvDsDisplayMeta *>(l_display->data);
      NvOSD_ArrowParams &arrow_params = display_meta->arrow_params[0];
      printf("arrow: x1: %d   y1: %d    x2: %d    y2: %d\n", arrow_params.x1, arrow_params.y1, arrow_params.x2, arrow_params.y2);
    }
  }

  return GST_PAD_PROBE_OK;
}

int main() {
  gst_init(nullptr, nullptr);
  setenv("USE_NEW_NVSTREAMMUX", "yes", 1);

  std::string pipeline_str =
      "videotestsrc ! nvvideoconvert ! video/x-raw(memory:NVMM),width=1280,height=720 ! mux.sink_0 "
      "videotestsrc ! nvvideoconvert ! video/x-raw(memory:NVMM),width=1280,height=720 ! mux.sink_1 "
      "videotestsrc ! nvvideoconvert ! video/x-raw(memory:NVMM),width=1280,height=720 ! mux.sink_2 "
      "videotestsrc ! nvvideoconvert ! video/x-raw(memory:NVMM),width=1280,height=720 ! mux.sink_3 "
      "nvstreammux name=mux batch-size=1 ! nvmultistreamtiler name=tiler ! nvvideoconvert ! nvdsosd ! "
#ifdef __aarch64__
      "nvegltransform ! "
#else
      "identity ! "
#endif
      "nveglglessink";
  GstElement *pipeline = gst_parse_launch(pipeline_str.c_str(), nullptr);
  GMainLoop *main_loop = g_main_loop_new(nullptr, false);
  GstBus *bus = gst_element_get_bus(pipeline);

  GstElement *streammux = gst_bin_get_by_name(GST_BIN(pipeline), "mux");
  GstPad *mux_src_pad = gst_element_get_static_pad(streammux, "src");
  gst_pad_add_probe(mux_src_pad, GST_PAD_PROBE_TYPE_BUFFER, mux_src_probe, nullptr, nullptr);
  gst_object_unref(mux_src_pad);

  GstElement *tiler = gst_bin_get_by_name(GST_BIN(pipeline), "tiler");
  GstPad *tiler_src_pad = gst_element_get_static_pad(tiler, "src");
  gst_pad_add_probe(tiler_src_pad, GST_PAD_PROBE_TYPE_BUFFER, tiler_src_probe, nullptr, nullptr);
  gst_object_unref(tiler_src_pad);

  auto callback_bus_message = +[] (GstBus *bus, GstMessage *msg, gpointer user_data) {
    GMainLoop *loop = static_cast<GMainLoop *>(user_data);
    switch (GST_MESSAGE_TYPE(msg)) {
      case GST_MESSAGE_EOS: {
        g_main_loop_quit(loop);
        break;
      }
      default:
        break;
    }
    return TRUE;
  };

  gst_bus_add_watch(bus, reinterpret_cast<GstBusFunc>(callback_bus_message), main_loop);
  gst_object_unref(bus);
  gst_element_set_state(pipeline, GST_STATE_PLAYING);
  g_main_loop_run(main_loop);
  gst_element_set_state(pipeline, GST_STATE_NULL);
  gst_object_unref(pipeline);
  g_main_loop_unref(main_loop);
}

We have 4 sources feed into nvstreammux.
In nvstreammux src pad, I attach a probe function to add a arrow-display meta with position: 200, 200, 300, 300
In nvmultistreamtiler src pad I attach a probe function to print rescaled-arrow-display meta. You can easily see that the arrow meta is rescaled properly on x86 platform

and on Jetson platform, it’s not rescaled properly

[yuweiw]

Which platform do you use？ I run your code in my Orin board with the same deepstream version(6.1.1) and JP(5.0.2) version. It works well.

[v.hunglx2]

It seems weird, I run it on my Xavier AGX dev kit with Jetpack 5.0.2 and DS 6.1.1 installed. Let me check it again

[v.hunglx2]

Do you have any Xavier board? I do not have any Orin board right now and can reproduce this issue on both NX and AGX dev kit.

[yuweiw]

OK, I will check it with the Xavier board. If there are any problems, we will analyze them as soon as possible.

[yuweiw]

There is no update from you for a period, assuming this is not an issue anymore.
Hence we are closing this topic. If need further support, please open a new one.
Thanks

I check it with the latest deepstream version, it works well. So could you help to update your deepstream version to the latest version?
https://developer.nvidia.com/deepstream_sdk_v6.1.1_jetson.tbz2