Issues extracting images from surface object

[st123010]

Please provide complete information as applicable to your setup.

• Hardware Platform (Jetson / GPU) GPU
• DeepStream Version 6.3 (Docker 6.3-triton-multiarch)
• JetPack Version (valid for Jetson only)
• TensorRT Version 8.4.1-1
• NVIDIA GPU Driver Version (valid for GPU only) 525.125.06
• Issue Type( questions, new requirements, bugs) questions

Hi, I tried to extract RGB images from surface objects by using the code below.
(Modified from Access frame pointer in deepstream-app - #24 by bcao). I found that the color format is NVBUF_COLOR_FORMAT_NV12_ER for stream sources but I don’t have any idea how to work with this particular color format and how to convert it’s to RGB image . I tried to used opencv but the images come out as grey-scaled and it’s channel seem a bit weird.

Testing with 1 rtsp stream sources with nvmultiurisrcbin (num_src = 1)

static GstPadProbeReturn
pgie_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info,
    gpointer u_data)
{
  
    cv::Mat in_mat;
    cv::Mat matBGR;

    GstBuffer *buf = (GstBuffer *) info->data;
    NvDsMetaList * l_frame = NULL;
    NvDsMetaList * l_user_meta = NULL;
    NvDsUserMeta *user_meta = NULL;
    NvDsInferSegmentationMeta *seg_meta_data = NULL;
    NvBufSurfaceColorFormat *color_format = NULL;

    // Get original raw data
    GstMapInfo in_map_info;
    char* src_data = NULL;
    if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
        g_print ("Error: Failed to map gst buffer\n");
        gst_buffer_unmap (buf, &in_map_info);
        return GST_PAD_PROBE_OK;
    }
    NvBufSurface *surface = (NvBufSurface *)in_map_info.data;

    NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);

    for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
      l_frame = l_frame->next) {
        NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
        /* Validate user meta */
        for (l_user_meta = frame_meta->frame_user_meta_list; l_user_meta != NULL;
            l_user_meta = l_user_meta->next) {
            user_meta = (NvDsUserMeta *) (l_user_meta->data);
            if (user_meta && user_meta->base_meta.meta_type == NVDSINFER_SEGMENTATION_META) {
                seg_meta_data = (NvDsInferSegmentationMeta*)user_meta->user_meta_data;
            }
        }

        src_data = (char*) malloc(surface->surfaceList[frame_meta->batch_id].dataSize);
        if(src_data == NULL) {
            g_print("Error: failed to malloc src_data \n");
            continue;
        }
#ifdef PLATFORM_TEGRA
        NvBufSurfaceMap (surface, -1, -1, NVBUF_MAP_READ);
        NvBufSurfacePlaneParams *pParams = &surface->surfaceList[frame_meta->batch_id].planeParams;
        unsigned int offset = 0;
        for(unsigned int num_planes=0; num_planes < pParams->num_planes; num_planes++){
            if(num_planes>0)
                offset += pParams->height[num_planes-1]*(pParams->bytesPerPix[num_planes-1]*pParams->width[num_planes-1]);
            for (unsigned int h = 0; h < pParams->height[num_planes]; h++) {
             memcpy((void *)(src_data+offset+h*pParams->bytesPerPix[num_planes]*pParams->width[num_planes]),
                    (void *)((char *)surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[num_planes]+h*pParams->pitch[num_planes]),
                    pParams->bytesPerPix[num_planes]*pParams->width[num_planes]
                    );
            }
        }
        NvBufSurfaceSyncForDevice (surface, -1, -1);
        NvBufSurfaceUnMap (surface, -1, -1);
#else
        cudaMemcpy((void*)src_data,
                   (void*)surface->surfaceList[frame_meta->batch_id].dataPtr,
                   surface->surfaceList[frame_meta->batch_id].dataSize,
                   cudaMemcpyDeviceToHost);
#endif

        // dump_jpg(src_data,
        //          surface->surfaceList[frame_meta->batch_id].width,
        //          surface->surfaceList[frame_meta->batch_id].height,
        //          seg_meta_data, frame_meta->source_id, frame_meta->frame_num);

        /*
        Color format from file sources: 34 -> NVBUF_COLOR_FORMAT_NV12_709_ER
        from stream sources: 7 -> NVBUF_COLOR_FORMAT_NV12_ER
        */

        if(src_data != NULL) {

            in_mat =
            cv::Mat (surface->surfaceList[frame_meta->batch_id].planeParams.height[0],
            surface->surfaceList[frame_meta->batch_id].planeParams.width[0], CV_8UC4,
            src_data,
            surface->surfaceList[frame_meta->batch_id].planeParams.pitch[0]);            

            cv::cvtColor(in_mat, matBGR, cv::COLOR_RGBA2RGB);
            //cv::cvtColor(in_mat, matBGR ,cv::COLOR_BGR2YUV_I420);

            if (save_img == 0)
            {
                imwrite("test.jpg", matBGR);
                save_img = 1;
            }
            
            free(src_data);
            src_data = NULL;
        }
    }
    gst_buffer_unmap (buf, &in_map_info);

    return GST_PAD_PROBE_OK;

}

The results from using the above code.

If replacing cvtColor function with this
cv::cvtColor(in_mat, matBGR ,cv::COLOR_BGR2YUV_I420);

any help is appreciated!

[miguel.taylor]

Hi @st123010,

NV12 is a type of planar YUV, you can use cv::COLOR_YUV2RGB_NV12 for the conversion.

[st123010]

Hello @miguel.taylor

I tried to change to cv::COLOR_YUV2RGB_NV12 as your suggestion but it’s won’t work. It’s seem likes there is something wrong with the image channel.

‘’’
terminate called after throwing an instance of ‘cv::Exception’
what(): OpenCV(4.2.0) …/modules/imgproc/src/color.simd_helpers.hpp:92: error: (-2:Unspecified error) in function ‘cv::impl::{anonymous}::CvtHelper<VScn, VDcn, VDepth, sizePolicy>::CvtHelper(cv::InputArray, cv::OutputArray, int) [with VScn = cv::impl::{anonymous}::Set<1>; VDcn = cv::impl::{anonymous}::Set<3, 4>; VDepth = cv::impl::{anonymous}::Set<0>; cv::impl::{anonymous}::SizePolicy sizePolicy = cv::impl::::FROM_YUV; cv::InputArray = const cv::_InputArray&; cv::OutputArray = const cv::_OutputArray&]’

Invalid number of channels in input image:
‘VScn::contains(scn)’
where
‘scn’ is 4
‘’’

[st123010]

I just found out that it’s possible to used NvBufSurfTransform to change the color format.