Configure different drawing in different video source

Hello,

I wish to configure a red box to video source0, blue box to video source1 and no box to video source2. From deepstream-test3-app, I add on NvDsLineMeta and set the drawing to different stream id. However, the stream id looks like not read one-by-one according to input sequence source0, source1 and source2. So, my drawing is shown on other stream sometimes. May I know how to overcome this?

Regards,
Nero

/*
 * Copyright (c) 2018 NVIDIA Corporation.  All rights reserved.
 *
 * NVIDIA Corporation and its licensors retain all intellectual property
 * and proprietary rights in and to this software, related documentation
 * and any modifications thereto.  Any use, reproduction, disclosure or
 * distribution of this software and related documentation without an express
 * license agreement from NVIDIA Corporation is strictly prohibited.
 *
 */

#include <gst/gst.h>
#include <glib.h>
#include <math.h>
#include <string.h>
#include <sys/time.h>

#include "gstnvdsmeta.h"
#include "gstnvstreammeta.h"
#include "gst-nvmessage.h"

#define MAX_DISPLAY_LEN 64

#define PGIE_CLASS_ID_VEHICLE 0
#define PGIE_CLASS_ID_PERSON 2

/* The muxer output resolution must be set if the input streams will be of
 * different resolution. The muxer will scale all the input frames to this
 * resolution. */
#define MUXER_OUTPUT_WIDTH 1920
#define MUXER_OUTPUT_HEIGHT 1080

/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set
 * based on the fastest source's framerate. */
#define MUXER_BATCH_TIMEOUT_USEC 4000000

#define TILED_OUTPUT_WIDTH 1920
#define TILED_OUTPUT_HEIGHT 1080

/* NVIDIA Decoder source pad memory feature. This feature signifies that source
 * pads having this capability will push GstBuffers containing cuda buffers. */
#define GST_CAPS_FEATURES_NVMM "memory:NVMM"

gchar pgie_classes_str[4][32] = {"Vehicle", "TwoWheeler", "Person",
                                 "RoadSign"};

#define FPS_PRINT_INTERVAL 300

static struct timeval start_time = {};

static guint probe_counter = 0;

/* tiler_sink_pad_buffer_probe  will extract metadata received on OSD sink pad
 * and update params for drawing rectangle, object information etc. */

static void generate_line_meta(gpointer data, guint id)
{
  guint num_lines;
  NvOSD_LineParams *l_params = NULL;

  NvDsLineMeta *meta = (NvDsLineMeta *)data;

  if (id == 0)
  {
    meta->num_lines = 4.0;
    meta->batch_id = 0.0;
    meta->line_params = (NvOSD_LineParams *)g_malloc0((meta->num_lines) * sizeof(NvOSD_LineParams));

    l_params = (NvOSD_LineParams *)&meta->line_params[0];
    l_params->x1 = 100;
    l_params->y1 = 100;
    l_params->x2 = 400;
    l_params->y2 = 150;
    l_params->line_width = 10;
    l_params->line_color.red = 1.0;
    l_params->line_color.green = 0.0;
    l_params->line_color.blue = 0.0;
    l_params->line_color.alpha = 0.5;

    l_params = (NvOSD_LineParams *)&meta->line_params[1];
    l_params->x1 = 400;
    l_params->y1 = 150;
    l_params->x2 = 500;
    l_params->y2 = 550;
    l_params->line_width = 10;
    l_params->line_color.red = 1.0;
    l_params->line_color.green = 0.0;
    l_params->line_color.blue = 0.0;
    l_params->line_color.alpha = 0.5;

    l_params = (NvOSD_LineParams *)&meta->line_params[2];
    l_params->x1 = 500;
    l_params->y1 = 550;
    l_params->x2 = 150;
    l_params->y2 = 500;
    l_params->line_width = 10;
    l_params->line_color.red = 1.0;
    l_params->line_color.green = 0.0;
    l_params->line_color.blue = 0.0;
    l_params->line_color.alpha = 0.5;

    l_params = (NvOSD_LineParams *)&meta->line_params[3];
    l_params->x1 = 150;
    l_params->y1 = 500;
    l_params->x2 = 100;
    l_params->y2 = 100;
    l_params->line_width = 10;
    l_params->line_color.red = 1.0;
    l_params->line_color.green = 0.0;
    l_params->line_color.blue = 0.0;
    l_params->line_color.alpha = 0.5;

  }
  
  if (id == 1)
  {
    meta->num_lines = 4.0;
    meta->batch_id = 1.0;
    meta->line_params = (NvOSD_LineParams *)g_malloc0((meta->num_lines) * sizeof(NvOSD_LineParams));

    l_params = (NvOSD_LineParams *)&meta->line_params[0];
    l_params->x1 = 100;
    l_params->y1 = 100;
    l_params->x2 = 400;
    l_params->y2 = 150;
    l_params->line_width = 10;
    l_params->line_color.red = 1.0;
    l_params->line_color.green = 1.0;
    l_params->line_color.blue = 0.0;
    l_params->line_color.alpha = 0.5;

    l_params = (NvOSD_LineParams *)&meta->line_params[1];
    l_params->x1 = 400;
    l_params->y1 = 150;
    l_params->x2 = 500;
    l_params->y2 = 550;
    l_params->line_width = 10;
    l_params->line_color.red = 1.0;
    l_params->line_color.green = 1.0;
    l_params->line_color.blue = 0.0;
    l_params->line_color.alpha = 0.5;

    l_params = (NvOSD_LineParams *)&meta->line_params[2];
    l_params->x1 = 500;
    l_params->y1 = 550;
    l_params->x2 = 150;
    l_params->y2 = 500;
    l_params->line_width = 10;
    l_params->line_color.red = 1.0;
    l_params->line_color.green = 1.0;
    l_params->line_color.blue = 0.0;
    l_params->line_color.alpha = 0.5;

    l_params = (NvOSD_LineParams *)&meta->line_params[3];
    l_params->x1 = 150;
    l_params->y1 = 500;
    l_params->x2 = 100;
    l_params->y2 = 100;
    l_params->line_width = 10;
    l_params->line_color.red = 1.0;
    l_params->line_color.green = 1.0;
    l_params->line_color.blue = 0.0;
    l_params->line_color.alpha = 0.5;

    //g_print("ID = %d, Yellow\n", id);
  }
}

static GstPadProbeReturn
tiler_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info,
                           gpointer u_data)
{

  GstMeta *gst_meta = NULL;
  NvDsMeta *nvdsmeta = NULL;
  gpointer state = NULL;
  static GQuark _nvdsmeta_quark = 0;
  GstBuffer *buf = (GstBuffer *)info->data;
  NvDsFrameMeta *frame_meta = NULL;
  guint num_rects = 0, rect_index = 0;
  NvDsObjectParams *obj_meta = NULL;
  NvOSD_TextParams *txt_params = NULL;
  GstNvStreamMeta *streammeta = NULL;

  if (!_nvdsmeta_quark)
    _nvdsmeta_quark = g_quark_from_static_string(NVDS_META_STRING);

  if (probe_counter == 0)
  {
    gettimeofday(&start_time, NULL);
  }
  else if (probe_counter == FPS_PRINT_INTERVAL)
  {
    struct timeval cur_time;
    gettimeofday(&cur_time, NULL);
    g_print("FPS for the last %d batches: %.2f\n", FPS_PRINT_INTERVAL,
            FPS_PRINT_INTERVAL / ((cur_time.tv_sec - start_time.tv_sec) +
                                  (cur_time.tv_usec - start_time.tv_usec) / 1e6));
    probe_counter = 0;
    start_time = cur_time;
  }
  probe_counter++;

  streammeta = gst_buffer_get_nvstream_meta(buf);

  while ((gst_meta = gst_buffer_iterate_meta(buf, &state)))
  {
    if (gst_meta_api_type_has_tag(gst_meta->info->api, _nvdsmeta_quark))
    {

      nvdsmeta = (NvDsMeta *)gst_meta;

      /* We are interested only in intercepting Meta of type
       * "NVDS_META_FRAME_INFO" as they are from our infer elements. */
      if (nvdsmeta->meta_type == NVDS_META_FRAME_INFO)
      {
        guint vehicle_count = 0;
        guint person_count = 0;

        frame_meta = (NvDsFrameMeta *)nvdsmeta->meta_data;
        if (frame_meta == NULL)
        {
          g_print("NvDS Meta contained NULL meta \n");
          return GST_PAD_PROBE_OK;
        }

        /* We reset the num_strings here as we plan to iterate through the
         *  the detected objects and form our own strings.
         *  The pipeline generated strings shall be discarded.
         */
        frame_meta->num_strings = 0;

        num_rects = frame_meta->num_rects;

        /* This means we have num_rects in frame_meta->obj_params,
         * now lets iterate through them */

        for (rect_index = 0; rect_index < num_rects; rect_index++)
        {
          /* Now using above information we need to form a text that should
           * be displayed on top of the bounding box, so lets form it here. */

          obj_meta = (NvDsObjectParams *)&frame_meta->obj_params[rect_index];

          txt_params = &(obj_meta->text_params);
          if (txt_params->display_text)
            g_free(txt_params->display_text);

          txt_params->display_text = g_malloc0(MAX_DISPLAY_LEN);

          g_snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "%s ",
                     pgie_classes_str[obj_meta->class_id]);

          if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE)
            vehicle_count++;
          if (obj_meta->class_id == PGIE_CLASS_ID_PERSON)
            person_count++;

          /* Now set the offsets where the string should appear */
          txt_params->x_offset = obj_meta->rect_params.left;
          txt_params->y_offset = obj_meta->rect_params.top - 25;

          /* Font , font-color and font-size */
          txt_params->font_params.font_name = "Arial";
          txt_params->font_params.font_size = 10;
          txt_params->font_params.font_color.red = 1.0;
          txt_params->font_params.font_color.green = 1.0;
          txt_params->font_params.font_color.blue = 1.0;
          txt_params->font_params.font_color.alpha = 1.0;

          /* Text background color */
          txt_params->set_bg_clr = 1;
          txt_params->text_bg_clr.red = 0.0;
          txt_params->text_bg_clr.green = 0.0;
          txt_params->text_bg_clr.blue = 0.0;
          txt_params->text_bg_clr.alpha = 1.0;

          frame_meta->num_strings++;
        }

        //g_print("Source ID = %d\n", frame_meta->stream_id);

        /* Enable this print by setting environment variable GST_DEBUG=4
         * before running the app. */
        GST_INFO("Source %d Frame Number = %lu Number of objects = %d "
                 "Vehicle Count = %d Person Count = %d",
                 frame_meta->stream_id,
                 streammeta->stream_frame_num[frame_meta->batch_id],
                 num_rects, vehicle_count, person_count);

NvDsMeta *gst_line_meta = NULL;

        NvDsLineMeta *line_meta = (NvDsLineMeta *)g_malloc0(sizeof(NvDsLineMeta));
        generate_line_meta(line_meta, frame_meta->stream_id);
        //g_print ("line = %d\n", line_meta->line_parasms->x1);
        gst_line_meta = gst_buffer_add_nvds_meta(buf, (void *)line_meta, (GDestroyNotify)nvds_free_line_info);
        if (gst_line_meta)
        {
          gst_line_meta->meta_type = NVDS_META_LINE_INFO;
        }
        else
        {
          g_print("Error in attaching event meta to buffer\n");
        }
      }
    }
  }
  return GST_PAD_PROBE_OK;
}

static gboolean
bus_call(GstBus *bus, GstMessage *msg, gpointer data)
{
  GMainLoop *loop = (GMainLoop *)data;
  switch (GST_MESSAGE_TYPE(msg))
  {
  case GST_MESSAGE_EOS:
    g_print("End of stream\n");
    g_main_loop_quit(loop);
    break;
  case GST_MESSAGE_WARNING:
  {
    gchar *debug;
    GError *error;
    gst_message_parse_warning(msg, &error, &debug);
    g_printerr("WARNING from element %s: %s\n",
               GST_OBJECT_NAME(msg->src), error->message);
    g_free(debug);
    g_printerr("Warning: %s\n", error->message);
    g_error_free(error);
    break;
  }
  case GST_MESSAGE_ERROR:
  {
    gchar *debug;
    GError *error;
    gst_message_parse_error(msg, &error, &debug);
    g_printerr("ERROR from element %s: %s\n",
               GST_OBJECT_NAME(msg->src), error->message);
    if (debug)
      g_printerr("Error details: %s\n", debug);
    g_free(debug);
    g_error_free(error);
    g_main_loop_quit(loop);
    break;
  }
  case GST_MESSAGE_ELEMENT:
  {
    if (gst_nvmessage_is_stream_eos(msg))
    {
      guint stream_id;
      if (gst_nvmessage_parse_stream_eos(msg, &stream_id))
      {
        g_print("Got EOS from stream %d\n", stream_id);
      }
    }
    break;
  }
  default:
    break;
  }
  return TRUE;
}

static void
cb_newpad(GstElement *decodebin, GstPad *decoder_src_pad, gpointer data)
{
  GstCaps *caps = gst_pad_query_caps(decoder_src_pad, NULL);
  const GstStructure *str = gst_caps_get_structure(caps, 0);
  const gchar *name = gst_structure_get_name(str);
  GstElement *source_bin = (GstElement *)data;
  GstCapsFeatures *features = gst_caps_get_features(caps, 0);

  /* Need to check if the pad created by the decodebin is for video and not
   * audio. */
  if (!strncmp(name, "video", 5))
  {
    /* Link the decodebin pad only if decodebin has picked nvidia
     * decoder plugin nvdec_*. We do this by checking if the pad caps contain
     * NVMM memory features. */
    if (gst_caps_features_contains(features, GST_CAPS_FEATURES_NVMM))
    {
      /* Get the source bin ghost pad */
      GstPad *bin_ghost_pad = gst_element_get_static_pad(source_bin, "src");
      if (!gst_ghost_pad_set_target(GST_GHOST_PAD(bin_ghost_pad),
                                    decoder_src_pad))
      {
        g_printerr("Failed to link decoder src pad to source bin ghost pad\n");
      }
      gst_object_unref(bin_ghost_pad);
    }
    else
    {
      g_printerr("Error: Decodebin did not pick nvidia decoder plugin.\n");
    }
  }
}

static GstElement *
create_source_bin(guint index, gchar *uri)
{
  GstElement *bin = NULL, *uri_decode_bin = NULL;
  gchar bin_name[16] = {};

  g_snprintf(bin_name, 15, "source-bin-%02d", index);

  /* Create a source GstBin to abstract this bin's content from the rest of the
   * pipeline */
  bin = gst_bin_new(bin_name);

  /* Source element for reading from the uri.
   * We will use decodebin and let it figure out the container format of the
   * stream and the codec and plug the appropriate demux and decode plugins. */
  uri_decode_bin = gst_element_factory_make("uridecodebin", "uri-decode-bin");

  if (!bin || !uri_decode_bin)
  {
    g_printerr("One element in source bin could not be created.\n");
    return NULL;
  }

  /* We set the input uri to the source element */
  g_object_set(G_OBJECT(uri_decode_bin), "uri", uri, NULL);

  /* Connect to the "pad-added" signal of the decodebin which generates a
   * callback once a new pad for raw data has beed created by the decodebin */
  g_signal_connect(G_OBJECT(uri_decode_bin), "pad-added",
                   G_CALLBACK(cb_newpad), bin);

  gst_bin_add(GST_BIN(bin), uri_decode_bin);

  /* We need to create a ghost pad for the source bin which will act as a proxy
   * for the video decoder src pad. The ghost pad will not have a target right
   * now. Once the decode bin creates the video decoder and generates the
   * cb_newpad callback, we will set the ghost pad target to the video decoder
   * src pad. */
  if (!gst_element_add_pad(bin, gst_ghost_pad_new_no_target("src",
                                                            GST_PAD_SRC)))
  {
    g_printerr("Failed to add ghost pad in source bin\n");
    return NULL;
  }

  return bin;
}

int main(int argc, char *argv[])
{
  GMainLoop *loop = NULL;
  GstElement *pipeline = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL,
             *nvvidconv = NULL, *nvosd = NULL, *tiler = NULL, *filter = NULL;
  GstBus *bus = NULL;
  guint bus_watch_id;
  GstPad *tiler_src_pad = NULL;
  guint i, num_sources;
  guint tiler_rows, tiler_columns;
  guint pgie_batch_size;

  /* Check input arguments */
  if (argc < 2)
  {
    g_printerr("Usage: %s <uri1> [uri2] ... [uriN] \n", argv[0]);
    return -1;
  }
  num_sources = argc - 1;

  /* Standard GStreamer initialization */
  gst_init(&argc, &argv);
  loop = g_main_loop_new(NULL, FALSE);

  /* Create gstreamer elements */
  /* Create Pipeline element that will form a connection of other elements */
  pipeline = gst_pipeline_new("dstest3-pipeline");

  /* Create nvstreammux instance to form batches from one or more sources. */
  streammux = gst_element_factory_make("nvstreammux", "stream-muxer");

  if (!pipeline || !streammux)
  {
    g_printerr("One element could not be created. Exiting.\n");
    return -1;
  }
  gst_bin_add(GST_BIN(pipeline), streammux);

  for (i = 0; i < num_sources; i++)
  {
    GstPad *sinkpad, *srcpad;
    gchar pad_name[16] = {};
    GstElement *source_bin = create_source_bin(i, argv[i + 1]);

    if (!source_bin)
    {
      g_printerr("Failed to create source bin. Exiting.\n");
      return -1;
    }

    gst_bin_add(GST_BIN(pipeline), source_bin);

    g_snprintf(pad_name, 15, "sink_%u", i);
    sinkpad = gst_element_get_request_pad(streammux, pad_name);
    if (!sinkpad)
    {
      g_printerr("Streammux request sink pad failed. Exiting.\n");
      return -1;
    }

    srcpad = gst_element_get_static_pad(source_bin, "src");
    if (!srcpad)
    {
      g_printerr("Failed to get src pad of source bin. Exiting.\n");
      return -1;
    }

    if (gst_pad_link(srcpad, sinkpad) != GST_PAD_LINK_OK)
    {
      g_printerr("Failed to link source bin to stream muxer. Exiting.\n");
      return -1;
    }

    gst_object_unref(srcpad);
    gst_object_unref(sinkpad);
  }

  /* Use nvinfer to infer on batched frame. */
  pgie = gst_element_factory_make("nvinfer", "primary-nvinference-engine");

  /* Use bboxfilter2 to infer on batched frame. */
  filter = gst_element_factory_make("bboxfilter2", "myfilter");

  /* Use nvtiler to composite the batched frames into a 2D tiled array based
   * on the source of the frames. */
  tiler = gst_element_factory_make("nvmultistreamtiler", "nvtiler");

  /* Use convertor to convert from NV12 to RGBA as required by nvosd */
  nvvidconv = gst_element_factory_make("nvvidconv", "nvvideo-converter");

  /* Create OSD to draw on the converted RGBA buffer */
  nvosd = gst_element_factory_make("nvosd", "nv-onscreendisplay");

  /* Finally render the osd output */
  sink = gst_element_factory_make("nveglglessink", "nvvideo-renderer");

  if (!pgie || !filter || !tiler || !nvvidconv || !nvosd || !sink)
  {
    g_printerr("One element could not be created. Exiting.\n");
    return -1;
  }

  g_object_set(G_OBJECT(streammux), "width", MUXER_OUTPUT_WIDTH, "height",
               MUXER_OUTPUT_HEIGHT, "batch-size", num_sources,
               "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL);

  /* Configure the nvinfer element using the nvinfer config file. */
  g_object_set(G_OBJECT(pgie),
               "config-file-path", "dstest3_pgie_config.txt", NULL);

  /* Override the batch-size set in the config file with the number of sources. */
  g_object_get(G_OBJECT(pgie), "batch-size", &pgie_batch_size, NULL);
  if (pgie_batch_size != num_sources)
  {
    g_printerr("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n",
               pgie_batch_size, num_sources);
    g_object_set(G_OBJECT(pgie), "batch-size", num_sources, NULL);
  }

  tiler_rows = (guint)sqrt(num_sources);
  tiler_columns = (guint)ceil(1.0 * num_sources / tiler_rows);
  /* we set the tiler properties here */
  g_object_set(G_OBJECT(tiler), "rows", tiler_rows, "columns", tiler_columns,
               "width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL);

  /* we set the osd properties here */
  g_object_set(G_OBJECT(nvosd), "font-size", 15, NULL);

  /* we add a message handler */
  bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline));
  bus_watch_id = gst_bus_add_watch(bus, bus_call, loop);
  gst_object_unref(bus);

  /* Set up the pipeline */
  /* we add all elements into the pipeline */
  gst_bin_add_many(GST_BIN(pipeline), pgie, filter, tiler, nvvidconv, nvosd, sink,
                   NULL);

  /* we link the elements together
   * nvstreammux -> nvinfer -> nvtiler -> nvvidconv -> nvosd -> video-renderer */
  if (!gst_element_link_many(streammux, pgie, filter, tiler, nvvidconv, nvosd, sink,
                             NULL))
  {
    g_printerr("Elements could not be linked. Exiting.\n");
    return -1;
  }

  /* Lets add probe to get informed of the meta data generated, we add probe to
   * the sink pad of the osd element, since by that time, the buffer would have
   * had got all the metadata. */
  tiler_src_pad = gst_element_get_static_pad(pgie, "src");
  if (!tiler_src_pad)
    g_print("Unable to get src pad\n");
  else
    gst_pad_add_probe(tiler_src_pad, GST_PAD_PROBE_TYPE_BUFFER,
                      tiler_src_pad_buffer_probe, NULL, NULL);

  /* Set the pipeline to "playing" state */
  g_print("Now playing:");
  for (i = 0; i < num_sources; i++)
  {
    g_print(" %s,", argv[i + 1]);
  }
  g_print("\n");
  gst_element_set_state(pipeline, GST_STATE_PLAYING);

  /* Wait till pipeline encounters an error or EOS */
  g_print("Running...\n");
  g_main_loop_run(loop);

  /* Out of the main loop, clean up nicely */
  g_print("Returned, stopping playback\n");
  gst_element_set_state(pipeline, GST_STATE_NULL);
  g_print("Deleting pipeline\n");
  gst_object_unref(GST_OBJECT(pipeline));
  g_source_remove(bus_watch_id);
  g_main_loop_unref(loop);
  return 0;
}


You are confused by 2 Ids.

The stream_id is your source id, but the batch_id is your frame index(batched frames) in current buffer.

you should modify your generate_line_meta function like this:

static void generate_line_meta(gpointer data, guint stream_id, guint batch_id)
{
     // set line color depend on stream_id
     // AND set meta->batch_id = batch_id

     // NOTE:  the stream_id is not always equal with batch_id
}

and then call this function like below:

generate_line_meta(line_meta, frame_meta->stream_id, frame_meta->batch_id);

if the frame sequence in GstBuffer like below:

source0,source0,source1,source1 (2 frames for each stream)

or

source2,source1,source0 (the sequence is not as same as stream id)

your original code will work fail, my code will work fine.

Hello zhouzhi9,

It works well. I am able to solve it now.

Thank you.