Saving Full Image Frame (Uncropped) with Bounding Box

[zenor87]

Please provide complete information as applicable to your setup.

• Hardware Platform (Jetson / GPU) Jetson
• DeepStream Version DS 5.1
• JetPack Version (valid for Jetson only) 4.5.1

Hi, I would like to know how can I save the full image frame (uncropped) with bounding box shown as a jpeg image. I’ve seen the deepstream_image_meta_test source code and am currently using the codes but it only saves the raw images without the bounding box.

Thanks!

[kesong]

Can you make one object meta data with the dimension of the full frame to save the full frame?

[zenor87]

hi kesong,

yes, my current codes modifies the object meta to save the full frame, the code is as below

static GstPadProbeReturn
pgie_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer ctx)
{
  GstBuffer *buf = (GstBuffer *) info->data;
  GstMapInfo inmap = GST_MAP_INFO_INIT;
  if (!gst_buffer_map (buf, &inmap, GST_MAP_READ)) {
    GST_ERROR ("input buffer mapinfo failed");
    return GST_FLOW_ERROR;
  }
  NvBufSurface *ip_surf = (NvBufSurface *) inmap.data;
  gst_buffer_unmap (buf, &inmap);

  NvDsObjectMeta *obj_meta = NULL;
  NvDsMetaList *l_frame = NULL;
  NvDsMetaList *l_obj = NULL;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);

  for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) 
  {
    NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);

    guint num_rects = 0;

    for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) 
    {
      obj_meta = (NvDsObjectMeta *) (l_obj->data);
      if (obj_meta->class_id == 0) {
        num_rects++;
      }
      /* Conditions that user needs to set to encode the detected objects of
       * interest. Here, by default all the detected objects are encoded.
       * For demonstration, we will encode the first object in the frame */
      if ( frame_meta->source_id == 0 && obj_meta->class_id == 0 && num_rects == 1 ) 
      { 
        NvDsObjEncUsrArgs userData = { 0 };
        /* To be set by user */
        userData.saveImg = TRUE;
        userData.attachUsrMeta = TRUE;
        /* Set if Image scaling Required */
        userData.scaleImg = FALSE;
        userData.scaledWidth = 0;
        userData.scaledHeight = 0;
        /* Preset */
        userData.objNum = num_rects;

        obj_meta->rect_params.width = frame_meta->source_frame_width ;
        obj_meta->rect_params.height = frame_meta->source_frame_height ;
        obj_meta->rect_params.top = 0.0f ;
        obj_meta->rect_params.left = 0.0f ;
        
        nvds_obj_enc_process (ctx, &userData, ip_surf, obj_meta, frame_meta);
      }
    }
  }

  nvds_obj_enc_finish (ctx);

  return GST_PAD_PROBE_OK;
}

both images saved in this function and in the osd_sink_pad_buffer_probe() function saves the full frame but without bounding boxes. would like to know how can I save the images with the bounding boxes.

[kesong]

BBox will draw onto video in nvdsosd. Please put the probe after nvdsosd.

[zenor87]

This is the function I referenced from the deepstream_image_meta_test source code

/* osd_sink_pad_buffer_probe will extract metadata received on OSD sink pad
 * and update params for drawing rectangle, object information. We also iterate
 * through the user meta of type "NVDS_CROP_IMAGE_META" to find image crop meta
 * and demonstrate how to access it.*/
static GstPadProbeReturn
osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info,
    gpointer u_data)
{
  GstBuffer *buf = (GstBuffer *) info->data;
  guint num_rects = 0;
  NvDsObjectMeta *obj_meta = NULL;
  guint vehicle_count = 0;
  guint person_count = 0;
  NvDsMetaList *l_frame = NULL;
  NvDsMetaList *l_obj = NULL;
  NvDsDisplayMeta *display_meta = NULL;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  int current_device = -1;
  cudaGetDevice(&current_device);
  struct cudaDeviceProp prop;
  cudaGetDeviceProperties(&prop, current_device);

  for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
      l_frame = l_frame->next) {
    NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
    int offset = 0;
    for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) {
      obj_meta = (NvDsObjectMeta *) (l_obj->data);
      if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) {
        vehicle_count++;
        num_rects++;
      }
      if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) {
        person_count++;
        num_rects++;
      }
      /* To verify  encoded metadata of cropped objects, we iterate through the
       * user metadata of each object and if a metadata of the type
       * 'NVDS_CROP_IMAGE_META' is found then we write that to a file as
       * implemented below.
       */
      char fileNameString[FILE_NAME_SIZE];
      const char *osd_string = "OSD";
      int obj_res_width = (int) obj_meta->rect_params.width;
      int obj_res_height = (int) obj_meta->rect_params.height;
      if(prop.integrated) {
        obj_res_width = GST_ROUND_DOWN_2(obj_res_width);
        obj_res_height = GST_ROUND_DOWN_2(obj_res_height);
      }

      snprintf (fileNameString, FILE_NAME_SIZE, "%s_%d_%d_%d_%s_%dx%d.jpg",
          osd_string, frame_number, frame_meta->source_id, num_rects,
          obj_meta->obj_label, obj_res_width, obj_res_height);
      /* For Demonstration Purposes we are writing metadata to jpeg images of
       * only vehicles for the first 100 frames only.
       * The files generated have a 'OSD' prefix. */
      if (frame_number < 100 && obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) {
        NvDsUserMetaList *usrMetaList = obj_meta->obj_user_meta_list;
        FILE *file;
        while (usrMetaList != NULL) {
          NvDsUserMeta *usrMetaData = (NvDsUserMeta *) usrMetaList->data;
          if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
            NvDsObjEncOutParams *enc_jpeg_image =
                (NvDsObjEncOutParams *) usrMetaData->user_meta_data;
            /* Write to File */
            file = fopen (fileNameString, "wb");
            fwrite (enc_jpeg_image->outBuffer, sizeof (uint8_t),
                enc_jpeg_image->outLen, file);
            fclose (file);
            usrMetaList = NULL;
          } else {
            usrMetaList = usrMetaList->next;
          }
        }
      }
    }
  }
  return GST_PAD_PROBE_OK;
}

Follow by the following code to add the probe

  NVGSTDS_BIN_ADD_GHOST_PAD (instance_bin->bin, last_elem, "sink");
  if (config->osd_config.enable) {
    NVGSTDS_ELEM_ADD_PROBE (instance_bin->all_bbox_buffer_probe_id,
        instance_bin->osd_bin.nvosd, "sink",
        osd_sink_pad_buffer_probe, GST_PAD_PROBE_TYPE_BUFFER, instance_bin);
  } else {
    NVGSTDS_ELEM_ADD_PROBE (instance_bin->all_bbox_buffer_probe_id,
        instance_bin->sink_bin.bin, "sink",
        osd_sink_pad_buffer_probe, GST_PAD_PROBE_TYPE_BUFFER, instance_bin);
  }

However, I’m still getting the raw images. Able to advise the proper way of doing?

[kesong]

sink pad of nvdsosd is the input buffer of nvdsosd. Please put probe function after that.