Save decoded frames for each stream (getting error while mapping the mermory)

Please provide complete information as applicable to your setup.

• Hardware Platform: 2080
• DeepStream Version: 5.0
• NVIDIA GPU Driver Version: 440.33.01
• Issue Type: requirement

Hi,
I want to access the decoded frames of each stream and save them. I followed the link

I added a probe to decoder src pad:

static GstPadProbeReturn

decode_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data){

    static int flag = 1;

    GstBuffer *buf = (GstBuffer *) info->data;

    NvDsBatchMeta *batch_meta =

        // gst_buffer_get_nvds_batch_meta (GST_BUFFER (info->data));

        gst_buffer_get_nvds_batch_meta (buf);

    GstMapInfo in_map_info;

    NvBufSurface *surface = NULL;

    memset (&in_map_info, 0, sizeof (in_map_info));

    if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {

        g_print ("Error: Failed to map gst buffer\n");

    }

    surface = (NvBufSurface *) in_map_info.data;

    int batch_size = surface->batchSize;

    printf("\nBatch Size : %d, resolution : %dx%d \n",batch_size,

        surface->surfaceList[0].width, surface->surfaceList[0].height);

    cudaError_t cuda_err;

    NvBufSurfTransformRect src_rect, dst_rect;

    src_rect.top   = 0;

    src_rect.left  = 0;

    src_rect.width = (guint) surface->surfaceList[0].width;

    src_rect.height= (guint) surface->surfaceList[0].height;

    dst_rect.top   = 0;

    dst_rect.left  = 0;

    dst_rect.width = (guint) surface->surfaceList[0].width;

    dst_rect.height= (guint) surface->surfaceList[0].height;

    NvBufSurfTransformParams nvbufsurface_params;

    nvbufsurface_params.src_rect = &src_rect;

    nvbufsurface_params.dst_rect = &dst_rect;

    nvbufsurface_params.transform_flag =  NVBUFSURF_TRANSFORM_CROP_SRC | NVBUFSURF_TRANSFORM_CROP_DST;

    nvbufsurface_params.transform_filter = NvBufSurfTransformInter_Default;

  

    NvBufSurface *dst_surface = NULL;

    NvBufSurfaceCreateParams nvbufsurface_create_params;

    /* An intermediate buffer for NV12/RGBA to BGR conversion  will be

     * required. Can be skipped if custom algorithm can work directly on NV12/RGBA. */

    nvbufsurface_create_params.gpuId  = surface->gpuId;

    nvbufsurface_create_params.width  = (gint) surface->surfaceList[0].width;

    nvbufsurface_create_params.height = (gint) surface->surfaceList[0].height;

    nvbufsurface_create_params.size = 0;

    nvbufsurface_create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;

    nvbufsurface_create_params.layout = NVBUF_LAYOUT_PITCH;

    nvbufsurface_create_params.memType = NVBUF_MEM_DEFAULT;

    cuda_err = cudaSetDevice (surface->gpuId);

    cudaStream_t cuda_stream;

    cuda_err=cudaStreamCreate (&cuda_stream);

    int create_result = NvBufSurfaceCreate(&dst_surface,batch_size,&nvbufsurface_create_params);    

    NvBufSurfTransformConfigParams transform_config_params;

    NvBufSurfTransform_Error err;

    transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;

    transform_config_params.gpu_id = surface->gpuId;

    transform_config_params.cuda_stream = cuda_stream;

    err = NvBufSurfTransformSetSessionParams (&transform_config_params);

    NvBufSurfaceMemSet (dst_surface, 0, 0, 0);

    err = NvBufSurfTransform (surface, dst_surface, &nvbufsurface_params);

    if (err != NvBufSurfTransformError_Success) {

      g_print ("NvBufSurfTransform failed with error %d while converting buffer\n", err);

    }

    NvBufSurfaceMap (dst_surface, 0, 0, NVBUF_MAP_READ);

    NvBufSurfaceSyncForCpu (dst_surface, 0, 0);

    cv::Mat bgr_frame = cv::Mat (cv::Size(nvbufsurface_create_params.width, nvbufsurface_create_params.height), CV_8UC3);

    cv::Mat in_mat =

        cv::Mat (nvbufsurface_create_params.height, nvbufsurface_create_params.width,

        CV_8UC4, dst_surface->surfaceList[0].mappedAddr.addr[0],

        dst_surface->surfaceList[0].pitch);

    cv::cvtColor (in_mat, bgr_frame, CV_RGBA2BGR);  

    NvBufSurfaceUnMap (dst_surface, 0, 0);

    NvBufSurfaceDestroy (dst_surface);

    cudaStreamDestroy (cuda_stream);

    gst_buffer_unmap (buf, &in_map_info);

    return GST_PAD_PROBE_OK;

}

But it throws the following error:

nvbufsurface: mapping of memory type (0) not supported
OpenCV Error: Assertion failed (total() == 0 || data != NULL) in Mat, file /usr/include/opencv2/core/mat.inl.hpp, line 431
terminate called after throwing an instance of 'cv::Exception'
  what():  /usr/include/opencv2/core/mat.inl.hpp:431: error: (-215) total() == 0 || data != NULL in function Mat

Aborted (core dumped)

I added nvvideoconvert setting nvbuf-memory-type:3 and probing its source pad but same error was thrown.

I also tried with the following code:

static GstPadProbeReturn

decode_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data){

    static int flag = 1;

    GstBuffer *buf = (GstBuffer *) info->data;

    NvDsBatchMeta *batch_meta =

        // gst_buffer_get_nvds_batch_meta (GST_BUFFER (info->data));

        gst_buffer_get_nvds_batch_meta (buf);

    GstMapInfo in_map_info;

    NvBufSurface *surface = NULL;

    memset (&in_map_info, 0, sizeof (in_map_info));

    if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {

        g_print ("Error: Failed to map gst buffer\n");

    }

    surface = (NvBufSurface *) in_map_info.data;

    int batch_size = surface->batchSize;

    printf("\nBatch Size : %d, resolution : %dx%d \n",batch_size,

        surface->surfaceList[0].width, surface->surfaceList[0].height);

    cv::Mat in_mat =

        cv::Mat (MUXER_OUTPUT_HEIGHT, MUXER_OUTPUT_WIDTH,

        CV_8UC4, surface->surfaceList[0].mappedAddr.addr[0],

        surface->surfaceList[0].pitch);

    cv::Mat out_mat;

    cv::cvtColor (in_mat, out_mat, CV_RGBA2BGR);

    if(flag == 1){ flag = 1; cv::imwrite("out_buf.jpg",out_mat); }

    

    NvBufSurfaceUnMap (surface, 0, 0);

    NvBufSurfaceDestroy (surface);

    gst_buffer_unmap (buf, &in_map_info);

    

    return GST_PAD_PROBE_OK;

} 

But it crashes after the first frame throwing following error:

Cuda failure: status=1 in CreateTextureObj at line 2496
nvbufsurftransform.cpp(2369) : getLastCudaError() CUDA error : Recevied NvBufSurfTransformError_Execution_Error : (1) invalid argument.

The output image does not save as proper format:

The modified deepstream_test1_app.c script:
deepstream_test_app_opencv.cpp (15.7 KB)

I used sample_720p.h264.

Thanks.

It needs some time to investigate your codes, will be back to you when we understand your requirement.

The attached code works. Please check with it. deepstream_test_app_opencv.cpp (15.7 KB)

Hi Fiona.Chen,
Thanks for the updated code. It code works as expected. I have a few queries I have.

  1. If multiple sources are mixed(using streammux) how do I get surface buffer for each of the sources?
  2. Copying the buffer from GPU to CPU seems to be costly reduces the throughput considerably. Is there a way to buffer the frames in GPU memory, and be later mapped to CPU?

Thanks.

You already know how to get data in NvBufSurface.
After nvstreammux, the frame buffers are managed by batch. You are just visit “surface->surfaceList[0]” in your code, right? After nvstreammux, “surface->surfaceList[1]” , “surface->surfaceList[2]”,… can also be visited based on your batch size.

Thanks, got that part of query. what about second query, is there a way we can allocate cuda buffer and keep the frames in GPU itself for later use

The buffer inside NvBufSurface is already GPU buffer which can be used by cuda.

There is sample in dsexample plugin source code. /opt/nvidia/deepstream/deepstream-5.0/sources/gst-plugins/gst-dsexample/

The NvBufSurface structure is defined in /opt/nvidia/deepstream/deepstream-5.0/sources/includes/nvbufsurface.h

https://docs.nvidia.com/metropolis/deepstream/sdk-api/Buf.html