How to save original frame data and convert OpenCv mat?before infer data

[908162104]

Please provide complete information as applicable to your setup.

• Hardware Platform (Jetson / GPU)
Jetson AGV
• DeepStream Version
6.1
• JetPack Version (valid for Jetson only)
5.1
• TensorRT Version
8.5.2

How to save original frame data and convert OpenCv mat and get infering after boundingbox and before infer data.
Thanks

[yuweiw]

Save original frame:You can refer to this: https://forums.developer.nvidia.com/t/deepstream-sdk-faq/80236/17
convert OpenCv mat: You need to learn how opencv converts the raw data.
We suggest you use NvBufSurfTransform to do transform.

[908162104]

I successfully get the scaled image and the inferred coordinates, but I also want to get the associated original image data. What should I do?

[yuweiw]

What’s your pipeline like? You can add the probe function to get the raw data before the nvinfer plugin.

[908162104]

If my use probe get queue,programme broken;

this pipline
  * -> video-renderer */
    if (!gst_element_link_many(streammux, queue1, pgie, queue2, nvdslogger, tiler,
                               queue3, nvvidconv, queue4, nvosd, queue5, sink, NULL))
    {
      g_printerr("Elements could not be linked. Exiting.\n");
      return -1;
    }

    /* Lets add probe to get informed of the meta data generated, we add probe to
     * the sink pad of the osd element, since by that time, the buffer would have
     * had got all the metadata. */
    tiler_src_pad = gst_element_get_static_pad(queue4, "src");
    if (!tiler_src_pad)
      g_print("Unable to get src pad\n");
    else
      gst_pad_add_probe(tiler_src_pad, GST_PAD_PROBE_TYPE_BUFFER,
                        tiler_src_pad_buffer_probe, NULL, NULL);
    gst_object_unref(tiler_src_pad);

this probe
static GstPadProbeReturn
tiler_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info,
                           gpointer u_data)
{
  GstBuffer *buf = (GstBuffer *)info->data;
  guint num_rects = 0;
  NvDsObjectMeta *obj_meta = NULL;
  guint vehicle_count = 0;
  guint person_count = 0;
  NvDsMetaList *l_frame = NULL;
  NvDsMetaList *l_obj = NULL;
  // NvDsDisplayMeta *display_meta = NULL;

  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf);
  auto ret = write_frame(buf);

  // g_print("add frame\n");
  // exit(0);
  return GST_PAD_PROBE_OK;
}

#define PLATFORM_TEGRA
int write_frame(GstBuffer *buf)
{
  NvDsMetaList *l_frame = NULL;
  NvDsMetaList *l_user_meta = NULL;
  // Get original raw data
  GstMapInfo in_map_info;
  char *src_data = NULL;
  if (!gst_buffer_map(buf, &in_map_info, GST_MAP_READ))
  {
    g_print("Error: Failed to map gst buffer\n");
    gst_buffer_unmap(buf, &in_map_info);
    return GST_PAD_PROBE_OK;
  }
  NvBufSurface *surface = (NvBufSurface *)in_map_info.data;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf);
  l_frame = batch_meta->frame_meta_list;
  if (l_frame)
  {
    NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data);
    /* Validate user meta */
    src_data = (char *)malloc(surface->surfaceList[frame_meta->batch_id].dataSize);
    if (src_data == NULL)
    {
      g_print("Error: failed to malloc src_data \n");
    }
#ifdef PLATFORM_TEGRA
    NvBufSurfaceMap(surface, -1, -1, NVBUF_MAP_READ);
    NvBufSurfacePlaneParams *pParams = &surface->surfaceList[frame_meta->batch_id].planeParams;

    unsigned int offset = 0;
    for (unsigned int num_planes = 0; num_planes < pParams->num_planes; num_planes++)
    {
      if (num_planes > 0)
        offset += pParams->height[num_planes - 1] * (pParams->bytesPerPix[num_planes - 1] * pParams->width[num_planes - 1]);
      for (unsigned int h = 0; h < pParams->height[num_planes]; h++)
      {
        memcpy((void *)(src_data + offset + h * pParams->bytesPerPix[num_planes] * pParams->width[num_planes]),
               (void *)((char *)surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[num_planes] + h * pParams->pitch[num_planes]),
               pParams->bytesPerPix[num_planes] * pParams->width[num_planes]);
      }
    }
    NvBufSurfaceSyncForDevice(surface, -1, -1);
    NvBufSurfaceUnMap(surface, -1, -1);
#else
    cudaMemcpy((void *)src_data,
               (void *)surface->surfaceList[frame_meta->batch_id].dataPtr,
               surface->surfaceList[frame_meta->batch_id].dataSize,
               cudaMemcpyDeviceToHost);
#endif

    gint frame_width = (gint)surface->surfaceList[frame_meta->batch_id].width;
    gint frame_height = (gint)surface->surfaceList[frame_meta->batch_id].height;
    gint frame_step = surface->surfaceList[frame_meta->batch_id].pitch;
    cv::Mat frame = cv::Mat(frame_height, frame_width, CV_8UC4, src_data, frame_step);
    // g_print("%d\n",frame.channels());
    // g_print("%d\n",frame.rows);
    // g_print("%d\n",frame.cols);

    cv::Mat out_mat = cv::Mat(cv::Size(frame_width, frame_height), CV_8UC3);
    cv::cvtColor(frame, out_mat, cv::COLOR_RGBA2BGR);
    bgr_buffer.push(out_mat.clone());

    // g_print("push a frame data\n");
    //  cv::imwrite("test.jpg", out_mat);

    if (src_data != NULL)
    {
      free(src_data);
      src_data = NULL;
    }
  }
  gst_buffer_unmap(buf, &in_map_info);
}

[908162104]

I try modify queue5 to queue1 and not convert frame data,but print frame width is after resize Dimensions ,not at it original size frame

[908162104]

[yuweiw]

There is no update from you for a period, assuming this is not an issue anymore. Hence we are closing this topic. If need further support, please open a new one. Thanks

You should add the probe function before streammux if you want to get original frame data.