In deepstream batch_id and source_id are not equal

I used QtGstreamer and deepstream plugin to build an app, the display was OK, but the boxes of objects jumped between sources. I finally found that in the “OnOsdSinkBinBufProbe” function ,the batch_id and source_id were not equal.

GstPadProbeReturn Camera::OnOsdSinkBinBufProbe(GstPad pad, GstPadProbeInfo info, gpointer u_data)
{
Camera
pCamera = (Camera
)u_data;
GstBuffer *buf = (GstBuffer *) info->data;

NvDsMetaList * l_frame = NULL;

if (!gst_buffer_is_writable (buf))
{
    return GST_PAD_PROBE_OK;
}
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
int nIndex = 0;
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
     l_frame = l_frame->next) {

    NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
    std::cout << "source_id: " << frame_meta->source_id << " batch_id: " << frame_meta->batch_id
                     << " surface_index: " << frame_meta->surface_index << " surface_type: " << frame_meta->surface_type << std::endl;
    pCamera->ProcessFrameMeta(frame_meta, batch_meta);

}
return GST_PAD_PROBE_OK;

}

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 2 surface_index: 0 surface_type: 0

source_id: 2 batch_id: 0 surface_index: 0 surface_type: 0

source_id: 0 batch_id: 1 surface_index: 0 surface_type: 0

source_id: 1 batch_id: 2 surface_index: 0 surface_type: 0

As a result, the box of a person original from source0 may appear in source1.

I have been stucked for a few days. Could someone tell me what was wrong.Thanks very much

Same issue here. The boxes jump.

Can you reproduce the issue by the sampes test1, test2, … or deepstream-app ?

You need not assume that the batch_id and source_id should be equal, batch_id indicates index of source_id in the current formed batch.

1 Like

test1 test2 and deepstream-app are all ok, but when integrated in Qt-Gstreamer , the boxes jumped or displayed together in one source

It’s hard to tell which part caused the issue without reproducing code, if possible can you share the test code?

I finally find that when the following codes are commented, that works.

//if (!gst_buffer_is_writable (buf))
//{
// return GST_PAD_PROBE_OK;
//}

Thanks very much

Thanks for sharing. glad to know solved.

Now it’s not clear how to get the correct surface to save in the source directory. I create directories for all sources and want to save crop to the appropriate directories. I assumed that the pointer to the surface in the surface->surfaceList[frame_meta->batch_id]/surface->surfaceList[frame_meta->source_id] was wrong.

static GstPadProbeReturn
tiler_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info,
    gpointer u_data)
{
    GstBuffer *buf = (GstBuffer *) info->data;
    guint num_rects = 0; 
    NvDsObjectMeta *obj_meta = NULL;
    guint vehicle_count = 0;
    guint person_count = 0;
    NvDsMetaList * l_frame = NULL;
    NvDsMetaList * l_obj = NULL;
    NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
    for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) {
      NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
      printf("source_id: %i batch_id: %i surface_index: %i surface_type: %i\n", frame_meta->source_id, frame_meta->batch_id,frame_meta->surface_index, frame_meta->surface_type);
      guint batch_id = frame_meta->source_id;
      int  surface_id = frame_meta->batch_id;
      for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) {
        guint class_id = 2555;
        obj_meta = (NvDsObjectMeta *) (l_obj->data);
        guint64 track_id = obj_meta->object_id;
        NvOSD_RectParams rect = obj_meta->rect_params;
        NvDsMetaList * l_class;// = obj_meta->classifier_meta_list;
        for (NvDsMetaList * l_class = obj_meta->classifier_meta_list; l_class != NULL; l_class = l_class->next) {
          NvDsClassifierMeta *cmeta = (NvDsClassifierMeta *) l_class->data;
          for (NvDsMetaList * l_label = cmeta->label_info_list; l_label != NULL; l_label = l_label->next) {
            NvDsLabelInfo *label = (NvDsLabelInfo *) l_label->data;
            class_id = label->result_class_id;
          }
        }   
        if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) {
            vehicle_count++;
            num_rects++;
        }
        if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) {
            person_count++;
            num_rects++;
        }
        gint min_side;
        NvBufSurfTransformRect src_rect, dst_rect;
        src_rect.top   = rect.top;
        src_rect.left  = rect.left;
        src_rect.width = rect.width;
        src_rect.height= rect.height;
        dst_rect.top   = 0;
        dst_rect.left  = 0;
        dst_rect.width = rect.width;
        dst_rect.height= rect.height;
        min_side = std::min(rect.width, rect.height);
        printf("source_id: %i, frame: %i, track_id: %i, class_id: %i\n", batch_id, frame_number, (int)track_id, class_id);
        if (frame_number%10 == 0 && min_side > 63){
          GstMapInfo in_map_info;
          NvBufSurface *surface = NULL;
          memset (&in_map_info, 0, sizeof (in_map_info));
          if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
            g_print ("Error: Failed to map gst buffer\n");
            gst_buffer_unmap (buf, &in_map_info);
            return GST_PAD_PROBE_OK;
          }
          cudaError_t cuda_err;
          surface = (NvBufSurface *) in_map_info.data; 
          int batch_size= surface->batchSize;
          NvBufSurfaceColorFormat color_format= surface->surfaceList[batch_id].colorFormat;
          NvBufSurfaceMemType memory_type = surface->memType;
          NvBufSurfTransformParams nvbufsurface_params;
          nvbufsurface_params.src_rect = &src_rect;
          nvbufsurface_params.dst_rect = &dst_rect;
          nvbufsurface_params.transform_flag =  NVBUFSURF_TRANSFORM_CROP_SRC | NVBUFSURF_TRANSFORM_CROP_DST;
          nvbufsurface_params.transform_filter = NvBufSurfTransformInter_Default;
          NvBufSurface *dst_surface = NULL;
          NvBufSurfaceCreateParams nvbufsurface_create_params;
          // An intermediate buffer for NV12/RGBA to BGR conversion  will be
          // required. Can be skipped if custom algorithm can work directly on NV12/RGBA. 
          nvbufsurface_create_params.gpuId  = surface->gpuId;
          nvbufsurface_create_params.width  = rect.width;
          nvbufsurface_create_params.height = rect.height;
          nvbufsurface_create_params.size = 0;
          nvbufsurface_create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
          nvbufsurface_create_params.layout = NVBUF_LAYOUT_PITCH;
          nvbufsurface_create_params.memType = NVBUF_MEM_CUDA_UNIFIED;
          cuda_err = cudaSetDevice (surface->gpuId);
          cudaStream_t cuda_stream;
          cuda_err=cudaStreamCreate (&cuda_stream);
          int create_result = NvBufSurfaceCreate(&dst_surface,batch_size,&nvbufsurface_create_params);  
          NvBufSurfTransformConfigParams transform_config_params;
          NvBufSurfTransform_Error err;
          transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
          transform_config_params.gpu_id = surface->gpuId;
          transform_config_params.cuda_stream = cuda_stream;
          err = NvBufSurfTransformSetSessionParams (&transform_config_params);
          NvBufSurfaceMemSet (dst_surface, 0, 0, 0);
          err = NvBufSurfTransform (surface, dst_surface, &nvbufsurface_params);
          if (err != NvBufSurfTransformError_Success) {
            g_print ("NvBufSurfTransform failed with error %d while converting buffer\n", err);
          }
          NvBufSurfaceMap (dst_surface, 0, 0, NVBUF_MAP_READ);
          NvBufSurfaceSyncForCpu (dst_surface, 0, 0);
          cv::Mat bgr_frame = cv::Mat (cv::Size(nvbufsurface_create_params.width, nvbufsurface_create_params.height), CV_8UC3);
          cv::Mat in_mat =
              cv::Mat (nvbufsurface_create_params.height, nvbufsurface_create_params.width,
              CV_8UC4, dst_surface->surfaceList[0].mappedAddr.addr[0],
              dst_surface->surfaceList[0].pitch);
          cv::cvtColor (in_mat, bgr_frame, cv::COLOR_RGBA2BGR);
          char first_dir[64];
          snprintf(first_dir, 64, "crops_%i", batch_id);
          if (create_dir(first_dir)!=0){
            printf("dir not created\n");
          }
          char sub_dir[64];
          snprintf(sub_dir, 64, "%s/%i",first_dir,track_id);
          
          if (create_dir(sub_dir)!=0){
            printf("dir not created\n");
          }
          char filename[64];
          snprintf(filename, 64, "%s/%i_%i.jpg",sub_dir, frame_number, class_id);
          cv::imwrite(filename,bgr_frame);
          NvBufSurfaceUnMap (dst_surface, 0, 0);
          NvBufSurfaceDestroy (dst_surface);
          cudaStreamDestroy (cuda_stream);
          gst_buffer_unmap (buf, &in_map_info);
        }
      }
          
    }
    frame_number++;
    return GST_PAD_PROBE_OK;
}

Hi,
Struct NvBufSurface field *surfaceList holds a pointer to an array of batched buffers, surface->>surfaceList[0] for source 0, surface->surfaceList[1] for source 1, surface->surfaceList[i] for source i.
you may refer to dsexample code, gst_dsexample_transform_ip -> get_converted_mat, to see how
each source buffer get processed. it should suite your needs.

Hi, I implemented a plugin based on dsexample and found out that that’s not allways true. For instance:

guint i = 0;
...
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next)
{
    frame_meta = (NvDsFrameMeta *)(l_frame->data);

    ...

    if ( frame_meta->source_id != i )
    {
        g_printerr("frame_meta->source_id != i (%d != %d)\n", frame_meta->source_id, i);
    }

    ...

    if (get_converted_mat (dsexample, frame_meta, surface, i, &rect_params,
			scale_ratio, dsexample->video_info.width,
			dsexample->video_info.height) != GST_FLOW_OK) {
		goto error;
    }
}

get_converted_mat uses the index i to reference surface->surfaceList[i]. It’s not a problem for dsexample, since it doesn’t need to save a state related to each source, but I did need to keep history of previous data for background subtraction.

You may need to have some mapping between source id and batch id.