Trying to save jpg images from 8 different rtsp streams but the frames extracted don't match

Accelerated Computing Intelligent Video Analytics DeepStream SDK
1

Hi. I’m running deepstream with 8 rtsp streams. I’m trying to save a jpg image from each stream every 30 seconds, however the frames saved and source id don’t match. Here is the code I used.

cv::Mat
  Mercury::getRGBFrame(GstMapInfo in_map_info, gint idx) {

    // To access the frame
    NvBufSurface *surface = NULL;
    NvBufSurface surface_idx;
    NvBufSurfTransformRect src_rect, dst_rect;

    surface = (NvBufSurface *) in_map_info.data;
    surface_idx = *surface;
    surface_idx.surfaceList = &(surface->surfaceList[idx]);
    surface_idx.numFilled = surface_idx.batchSize = 1;

    int batch_size = surface_idx.batchSize;
    src_rect.top   = 0;
    src_rect.left  = 0;
    src_rect.width = (guint) surface->surfaceList[idx].width;
    src_rect.height= (guint) surface->surfaceList[idx].height;

    dst_rect.top   = 0;
    dst_rect.left  = 0;
    dst_rect.width = (guint) surface->surfaceList[idx].width;
    dst_rect.height= (guint) surface->surfaceList[idx].height;

    NvBufSurfTransformParams nvbufsurface_params;
    nvbufsurface_params.src_rect = &src_rect;
    nvbufsurface_params.dst_rect = &dst_rect;
    nvbufsurface_params.transform_flag =  NVBUFSURF_TRANSFORM_CROP_SRC | NVBUFSURF_TRANSFORM_CROP_DST;
    nvbufsurface_params.transform_filter = NvBufSurfTransformInter_Default;

    NvBufSurface *dst_surface = NULL;
    NvBufSurfaceCreateParams nvbufsurface_create_params;

    nvbufsurface_create_params.gpuId  = surface->gpuId;
    nvbufsurface_create_params.width  = (guint) surface->surfaceList[idx].width;
    nvbufsurface_create_params.height = (guint) surface->surfaceList[idx].height;
    nvbufsurface_create_params.size = 0;
    // nvbufsurface_create_params.isContiguous = true;
    nvbufsurface_create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
    nvbufsurface_create_params.layout = NVBUF_LAYOUT_PITCH;

    // THE memType PARAM IS SET TO CUDA UNIFIED IN dGPU DEVICES COMMENT IT out
    // AND USE THE IMMEDIATE NEXT LINE TO SET THE memType PARAM FOR JETSON DEVICES

    #ifdef PLATFORM_TEGRA
      nvbufsurface_create_params.memType = NVBUF_MEM_DEFAULT;
    #else
      nvbufsurface_create_params.memType = NVBUF_MEM_CUDA_UNIFIED;
    #endif

    cudaError_t cuda_err = cudaSetDevice (surface->gpuId);

    cudaStream_t cuda_stream;

    cuda_err = cudaStreamCreate(&cuda_stream);

    int create_result = NvBufSurfaceCreate(&dst_surface, batch_size, &nvbufsurface_create_params);

    NvBufSurfTransformConfigParams transform_config_params;

    NvBufSurfTransform_Error err;

    transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
    transform_config_params.gpu_id = surface->gpuId;
    transform_config_params.cuda_stream = cuda_stream;
    err = NvBufSurfTransformSetSessionParams (&transform_config_params);

    NvBufSurfaceMemSet (dst_surface, 0, 0, 0);

    err = NvBufSurfTransform (&surface_idx, dst_surface, &nvbufsurface_params);

    if (err != NvBufSurfTransformError_Success) {
      g_print ("NvBufSurfTransform failed with error %d while converting buffer\n", err);
    }

    NvBufSurfaceMap (dst_surface, 0, 0, NVBUF_MAP_READ);
    NvBufSurfaceSyncForCpu (dst_surface, 0, 0);

    cv::Mat bgr_frame = cv::Mat (cv::Size(nvbufsurface_create_params.width, nvbufsurface_create_params.height), CV_8UC3);

    cv::Mat in_mat = cv::Mat (nvbufsurface_create_params.height, nvbufsurface_create_params.width, CV_8UC4, dst_surface->surfaceList[0].mappedAddr.addr[0], dst_surface->surfaceList[0].pitch);

    #if (CV_VERSION_MAJOR >= 4)
      cv::cvtColor (in_mat, bgr_frame, COLOR_RGBA2BGR);
    #else
      cv::cvtColor (in_mat, bgr_frame, CV_RGBA2BGR);
    #endif

    NvBufSurfaceUnMap(dst_surface, 0 , 0);
    NvBufSurfaceUnMap(&surface_idx, 0 , 0);


    NvBufSurfaceDestroy(dst_surface);

    cudaStreamDestroy (cuda_stream);
    
    return bgr_frame;
  }
2

What’s the streammux batch-size you set? The source id is assigned when source src-pad link to streammux sink-pad in order.

3

@ChrisDing the streammux batch size is 8 as well. I’m basically building an intrusion system. I have a probe at the sink of tiler plugin and I’m accurately able to access the frame_meta and object_meta per source_id. It’s just the image extraction in jpg that seems to be random.

4

Hi. Could you please give me something I could try to fix my issue?

5

Can you give the pipeline: rtsp → dec → streammux → pgie → osd → tiler → … ?

6

Need to make sure that frames extracted from Metadata and nvbufsurface list belong to the same source.

7

We haven’t heard back from you in a couple weeks, so marking this topic closed.
Please open a new forum issue when you are ready and we’ll pick it up there.