RTSP camera access frame issue

Hi Guys,

I am trying to run deepstream-app using RTSP. I am trying to access the frame in the tracking probe function in deepstream-app. I referred an earlier post :

https://devtalk.nvidia.com/default/topic/1058086/deepstream-sdk/how-to-run-rtp-camera-in-deepstream-on-nano/post/5374027/#5374027

From this post, I could understand that the color format of the frame is NvBufferColorFormat_NV12_709_ER which is YUV420 multi-planar.However, I am unable to understand the dataSize, pitch, width and height values got from NvBufSurfaceParams.

/**
 * Buffer probe function after tracker.
 */
static GstPadProbeReturn
tracking_done_buf_prob (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
  NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
  guint index = bin->index;
  AppCtx *appCtx = bin->appCtx;
  GstBuffer *buf = (GstBuffer *) info->data;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  if (!batch_meta) {
    NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
    return GST_PAD_PROBE_OK;
  }

  GstMapInfo in_map_info;
  NvBufSurface *surface = NULL;

  memset (&in_map_info, 0, sizeof (in_map_info));
  if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
    g_print ("Error: Failed to map gst buffer\n");
  }
  surface = (NvBufSurface *) in_map_info.data;  

  int batch_size= surface->batchSize;

  for(int i=0; i<batch_size; ++i)
  {
	uint32_t data_size =  surface->surfaceList[i].dataSize;
	uint32_t pitch =  surface->surfaceList[i].pitch;
	uint32_t width =  surface->surfaceList[i].width;
	uint32_t height =  surface->surfaceList[i].height;

	void *dataPtr = surface->surfaceList[i].dataPtr;
	
	printf("Size of the frame buffer : %d\n",data_size);
	printf("Pitch of the frame buffer : %d\n",pitch);
	printf("width of the frame buffer : %d\n",width);
	printf("height of the frame buffer : %d\n",height);

	NvBufSurfaceColorFormat color_format= surface->surfaceList[i].colorFormat;

        if (color_format == NVBUF_COLOR_FORMAT_NV12)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_ER \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709_ER \n");
  }

  /*
   * Output KITTI labels with tracking ID if configured to do so.
   */
  write_kitti_track_output(appCtx, batch_meta);

  if (appCtx->primary_bbox_generated_cb)
    appCtx->primary_bbox_generated_cb (appCtx, buf, batch_meta, index);
  return GST_PAD_PROBE_OK;
}

Following is the output :

Size of the frame buffer : 1572864
Pitch of the frame buffer : 1280
width of the frame buffer : 1280
height of the frame buffer : 720

I am unable to comprehend what is going on. Kindly let me know where I am going wrong or is there any information which I am missing out on.

Thanks.

Hi Guys,

I tried to dump other information to get more insights. Please find the code below and the dumps:

Code :

/**
 * Buffer probe function after tracker.
 */
static GstPadProbeReturn
tracking_done_buf_prob (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
  NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
  guint index = bin->index;
  AppCtx *appCtx = bin->appCtx;
  GstBuffer *buf = (GstBuffer *) info->data;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  if (!batch_meta) {
    NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
    return GST_PAD_PROBE_OK;
  }

  GstMapInfo in_map_info;
  NvBufSurface *surface = NULL;

  memset (&in_map_info, 0, sizeof (in_map_info));
  if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
    g_print ("Error: Failed to map gst buffer\n");
  }
  surface = (NvBufSurface *) in_map_info.data;  

  int batch_size= surface->batchSize;

  for(int i=0; i<batch_size; ++i)
  {
	uint32_t data_size =  surface->surfaceList[i].dataSize;
	uint32_t pitch =  surface->surfaceList[i].pitch;
	uint32_t width =  surface->surfaceList[i].width;
	uint32_t height =  surface->surfaceList[i].height;
	NvBufSurfaceLayout layout = surface->surfaceList[i].layout;
	NvBufSurfacePlaneParams plane_params = surface->surfaceList[i].planeParams;
 	void *dataPtr = surface->surfaceList[i].dataPtr;
	
	uint32_t num_planes=plane_params.num_planes;
	
	printf("\nNumber of Planes : %d\n\n", num_planes);

	for(int i=0 ; i < num_planes; ++i)
	{
	   uint32_t width=plane_params.width[i];
	   uint32_t height=plane_params.height[i];
	   uint32_t pitch=plane_params.pitch[i]; 
           uint32_t offset=plane_params.offset[i];
           uint32_t psize=plane_params.psize[i];
           uint32_t bytes_per_pix=plane_params.bytesPerPix[i];
		
	   printf("width of the plane %d : %d\n\n",i,width);
	   printf("height of the plane %d : %d\n\n",i,height);
	   printf("pitch of the plane  %d : %d\n\n",i,pitch);
	   printf("offset of the plane  %d : %d\n\n",i,offset);
	   printf("psize of the plane  %d : %d\n\n",i,psize);
	   printf("bytes_per_pix of the plane  %d : %d\n\n\n\n",i,bytes_per_pix);
	   
	}


	printf("Size of the frame buffer : %d\n\n",data_size);
	printf("Pitch of the frame buffer : %d\n\n",pitch);
	printf("width of the frame buffer : %d\n\n",width);
	printf("height of the frame buffer : %d\n\n",height);

	NvBufSurfaceColorFormat color_format= surface->surfaceList[i].colorFormat;

        if (color_format == NVBUF_COLOR_FORMAT_NV12)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_ER \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709_ER \n");
  }

  /*
   * Output KITTI labels with tracking ID if configured to do so.
   */
  write_kitti_track_output(appCtx, batch_meta);

  if (appCtx->primary_bbox_generated_cb)
    appCtx->primary_bbox_generated_cb (appCtx, buf, batch_meta, index);
  return GST_PAD_PROBE_OK;
}

Dumps:

Number of Planes : 2

width of the plane 0 : 1280
height of the plane 0 : 720
pitch of the plane  0 : 1280
offset of the plane  0 : 0
psize of the plane  0 : 1048576
bytes_per_pix of the plane  0 : 1

width of the plane 1 : 640
height of the plane 1 : 360
pitch of the plane  1 : 1280
offset of the plane  1 : 1048576
psize of the plane  1 : 524288
bytes_per_pix of the plane  1 : 2

Size of the frame buffer : 1572864
Pitch of the frame buffer : 1280
width of the frame buffer : 1280
height of the frame buffer : 720

I am still unable to understand the values of psize and dataSize.

Kindly help me out.

Thanks.

Hi Guys,

I get segmentation fault midway when I just try to access the Y channel buffer. Also, I am unable to understand the dumps I get by printing the parameters.

Code:

void parse_ychannel_row_major(uint32_t width, uint32_t height,const uint8_t *Y,uint32_t Y_stride)
{
	int x,y;
	for(y=0; y< height; ++y)
	{
		const uint8_t *y_row_ptr=Y+y*Y_stride;

		for(x=0;x<width;++x)
		{
			uint8_t data = *(y_row_ptr+x);
			printf("Y Pixel Value : %d\n",data);
			printf("x : %d , y : %d \n",x,y);
		}

	}

}

static GstPadProbeReturn
tracking_done_buf_prob (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
  NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
  guint index = bin->index;
  AppCtx *appCtx = bin->appCtx;
  GstBuffer *buf = (GstBuffer *) info->data;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  if (!batch_meta) {
    NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
    return GST_PAD_PROBE_OK;
  }

  GstMapInfo in_map_info;
  NvBufSurface *surface = NULL;

  memset (&in_map_info, 0, sizeof (in_map_info));
  if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
    g_print ("Error: Failed to map gst buffer\n");
  }

#if 1

  surface = (NvBufSurface *) in_map_info.data;  

  int batch_size= surface->batchSize;

  printf("Batch Size : %d",batch_size );

for(int i=0; i<batch_size; ++i)
  {
	uint32_t data_size =  surface->surfaceList[i].dataSize;
	uint32_t pitch =  surface->surfaceList[i].pitch;
	uint32_t width =  surface->surfaceList[i].width;
	uint32_t height =  surface->surfaceList[i].height;
	NvBufSurfaceLayout layout = surface->surfaceList[i].layout;
	NvBufSurfacePlaneParams plane_params = surface->surfaceList[i].planeParams;
 	void *dataPtr = surface->surfaceList[i].dataPtr;
	uint8_t *data = dataPtr;
	
 	//printf("\nData at first index buffer %d : %d",i,*data);

	uint32_t num_planes=plane_params.num_planes;
	
	printf("\nNumber of Planes : %d\n\n", num_planes);

	for(int i=0 ; i < num_planes; ++i)
	{
	   uint32_t width=plane_params.width[i];
	   uint32_t height=plane_params.height[i];
	   uint32_t pitch=plane_params.pitch[i]; 
           uint32_t offset=plane_params.offset[i];
           uint32_t psize=plane_params.psize[i];
           uint32_t bytes_per_pix=plane_params.bytesPerPix[i];
		
	   printf("width of the plane %d : %d\n\n",i,width);
	   printf("height of the plane %d : %d\n\n",i,height);
	   printf("pitch of the plane  %d : %d\n\n",i,pitch);
	   printf("offset of the plane  %d : %d\n\n",i,offset);
	   printf("psize of the plane  %d : %d\n\n",i,psize);
	   printf("bytes_per_pix of the plane  %d : %d\n\n\n\n",i,bytes_per_pix);	   
	}

	printf("Size of the frame buffer : %d\n\n",data_size);
	printf("Pitch of the frame buffer : %d\n\n",pitch);
	printf("width of the frame buffer : %d\n\n",width);
	printf("height of the frame buffer : %d\n\n",height);

	NvBufSurfaceColorFormat color_format= surface->surfaceList[i].colorFormat;

        if (color_format == NVBUF_COLOR_FORMAT_NV12)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_ER \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709_ER \n");
  }

uint32_t frame_width=surface->surfaceList[0].planeParams.width[0];
   uint32_t frame_height=surface->surfaceList[0].planeParams.height[0];
   uint32_t Y_stride=surface->surfaceList[0].planeParams.pitch[0];
   uint32_t UV_stride=surface->surfaceList[0].planeParams.pitch[1];

   printf("\n\nframe_width : %d\n\n", frame_width);
   printf("\n\nframe_height : %d\n\n", frame_height);
   printf("\n\nY_stride : %d\n\n", Y_stride);
   printf("\n\nUV_stride : %d\n\n", UV_stride);

int offset_calc=frame_width*frame_height;

   //Method 1
   const uint8_t *Y=surface->surfaceList[0].dataPtr;

   // Method 2 
   //const uint8_t *Y=surface->surfaceList[0].dataPtr + (surface->surfaceList[0].planeParams.psize[0] - offset_calc);
   
   parse_ychannel_row_major(frame_width, frame_height,Y,Y_stride);

  exit(0); 

#endif
  /*
   * Output KITTI labels with tracking ID if configured to do so.
   */
  write_kitti_track_output(appCtx, batch_meta);

  if (appCtx->primary_bbox_generated_cb)
    appCtx->primary_bbox_generated_cb (appCtx, buf, batch_meta, index);
  return GST_PAD_PROBE_OK;
}

Beginning of the log with the parameters :

Creating LL OSD context new
KLT Tracker Init
Batch Size : 1
Number of Planes : 2

width of the plane 0 : 1280
height of the plane 0 : 720
pitch of the plane  0 : 1280
offset of the plane  0 : 0
psize of the plane  0 : 1048576
bytes_per_pix of the plane  0 : 1

width of the plane 1 : 640
height of the plane 1 : 360
pitch of the plane  1 : 1280
offset of the plane  1 : 1048576
psize of the plane  1 : 524288
bytes_per_pix of the plane  1 : 2

Size of the frame buffer : 1572864
Pitch of the frame buffer : 1280
width of the frame buffer : 1280
height of the frame buffer : 720

color_format: NVBUF_COLOR_FORMAT_NV12_709_ER

frame_width : 1280
frame_height : 720
Y_stride : 1280
UV_stride : 1280

The end of the log looks like the following before it crashes:

Y Pixel Value : 0
x : 615 , y : 286 
Y Pixel Value : 0
x : 616 , y : 286 
Y Pixel Value : 0
x : 617 , y : 286 
Y Pixel Value : 0
x : 618 , y : 286 
Y Pixel Value : 0
x : 619 , y : 286 
Y Pixel Value : 0
x : 620 , y : 286 
Y Pixel Value : 0
x : 621 , y : 286 
Y Pixel Value : 0
x : 622 , y : 286 
Y Pixel Value : 0
x : 623 , y : 286 
Y Pixel Value : 0
x : 624 , y : 286 
Y Pixel Value : 0
x : 625 , y : 286 
Y Pixel Value : 0
x : 626 , y : 286 
Y Pixel Value : 0
x : 627 , y : 286 
Y Pixel Value : 0
x : 628 , y : 286 
Y Pixel Value : 0
x : 629 , y : 286 
Y Pixel Value : 0
x Segmentation fault (core dumped)

The psize of the plane is greater than the product of width and height. Is there anything wrong with the way I am accessing the frame? Is there a possible memory access conflict?

Kindly help me out.

Thanks.

Pitch of the plane @1280 should always be in relation to a 720x1280 resolution.
I see you have 640x360. Try 640x480 and pitch of the plane @640 or maybe 720 but not 1280.
Try to alter the height and width.

Hi Guys,

I realised I needed to map the frame buffer pointer to CPU address space using “NvBufSurfaceMap”. Doing so, I have eliminated the problem of segmentation fault. However, when I dump the Y channel buffer using libjpeg, I do not get the right output (image attached). Please find the code below:

int write_jpeg_file( char *filename, unsigned char* rgb_image , int width, int height, int bytes_per_pixel, J_COLOR_SPACE color_space )
{

	struct jpeg_compress_struct cinfo;
	struct jpeg_error_mgr jerr;
	
	JSAMPROW row_pointer[1];
	FILE *outfile = fopen( filename, "wb" );
	
	if ( !outfile )
	{
		printf("Error opening output jpeg file %s\n!", filename );
		return -1;
	}
	cinfo.err = jpeg_std_error( &jerr );
	jpeg_create_compress(&cinfo);
	jpeg_stdio_dest(&cinfo, outfile);

	cinfo.image_width = width;	
	cinfo.image_height = height;
	cinfo.input_components = bytes_per_pixel;
	cinfo.in_color_space = color_space; //JCS_RGB

	jpeg_set_defaults( &cinfo );

	jpeg_start_compress( &cinfo, TRUE );

	while( cinfo.next_scanline < cinfo.image_height )
	{
		row_pointer[0] = &rgb_image[ cinfo.next_scanline * cinfo.image_width *  cinfo.input_components];
		jpeg_write_scanlines( &cinfo, row_pointer, 1 );
	}

	jpeg_finish_compress( &cinfo );
	jpeg_destroy_compress( &cinfo );
	fclose( outfile );

	return 1;
}

/**
 * Buffer probe function after tracker.
 */
static GstPadProbeReturn
tracking_done_buf_prob (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
  NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
  guint index = bin->index;
  AppCtx *appCtx = bin->appCtx;
  GstBuffer *buf = (GstBuffer *) info->data;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  if (!batch_meta) {
    NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
    return GST_PAD_PROBE_OK;
  }

  GstMapInfo in_map_info;
  NvBufSurface *surface = NULL;

  memset (&in_map_info, 0, sizeof (in_map_info));
  if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
    g_print ("Error: Failed to map gst buffer\n");
  }

#if 1

  surface = (NvBufSurface *) in_map_info.data;  

  int batch_size= surface->batchSize;

  printf("Batch Size : %d",batch_size );

  int map_result = NvBufSurfaceMap(surface,-1,-1,NVBUF_MAP_READ);	   
  printf("\nMap Result : %d\n", map_result);

  int sync_result = NvBufSurfaceSyncForCpu(surface,-1,-1);
  printf("\nSync Result : %d\n", sync_result);

  for(int i=0; i<batch_size; ++i)
  {
	uint32_t data_size =  surface->surfaceList[i].dataSize;
	uint32_t pitch =  surface->surfaceList[i].pitch;
	uint32_t width =  surface->surfaceList[i].width;
	uint32_t height =  surface->surfaceList[i].height;
	NvBufSurfaceLayout layout = surface->surfaceList[i].layout;
	NvBufSurfacePlaneParams plane_params = surface->surfaceList[i].planeParams;
 	void *dataPtr = surface->surfaceList[i].dataPtr;
	printf("\nSize of the first location : %d\n", sizeof(dataPtr[0]));
	uint8_t* data = dataPtr;
	
	printf("\n\nData pointer : %p\n",data);

	uint32_t num_planes=plane_params.num_planes;
	
	printf("\nNumber of Planes : %d\n\n", num_planes);

	for(int i=0 ; i < num_planes; ++i)
	{
	   uint32_t width=plane_params.width[i];
	   uint32_t height=plane_params.height[i];
	   uint32_t pitch=plane_params.pitch[i]; 
           uint32_t offset=plane_params.offset[i];
           uint32_t psize=plane_params.psize[i];
           uint32_t bytes_per_pix=plane_params.bytesPerPix[i];
		
	   printf("width of the plane %d : %d\n\n",i,width);
	   printf("height of the plane %d : %d\n\n",i,height);
	   printf("pitch of the plane  %d : %d\n\n",i,pitch);
	   printf("offset of the plane  %d : %d\n\n",i,offset);
	   printf("psize of the plane  %d : %d\n\n",i,psize);
	   printf("bytes_per_pix of the plane  %d : %d\n\n\n\n",i,bytes_per_pix);	   
	}

	printf("Size of the frame buffer : %d\n\n",data_size);
	printf("Pitch of the frame buffer : %d\n\n",pitch);
	printf("width of the frame buffer : %d\n\n",width);
	printf("height of the frame buffer : %d\n\n",height);

	NvBufSurfaceColorFormat color_format= surface->surfaceList[i].colorFormat;

        if (color_format == NVBUF_COLOR_FORMAT_NV12)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_ER \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709_ER \n");

uint32_t frame_width=surface->surfaceList[i].planeParams.width[0];
   uint32_t frame_height=surface->surfaceList[i].planeParams.height[0];
   uint32_t Y_stride=surface->surfaceList[i].planeParams.pitch[0];
   uint32_t UV_stride=surface->surfaceList[i].planeParams.pitch[1];

   printf("\n\nframe_width : %d\n\n", frame_width);
   printf("\n\nframe_height : %d\n\n", frame_height);
   printf("\n\nY_stride : %d\n\n", Y_stride);
   printf("\n\nUV_stride : %d\n\n", UV_stride);

   uint8_t *RGB= malloc(3*frame_width*frame_height);
   uint32_t RGB_stride=frame_width*3;
   YCbCrType yuv_type=YCBCR_709;
 
   printf("\n\nData at first index of RGB : %d\n\n", *RGB);
   printf("\n\nRGB_stride : %d\n\n", RGB_stride);

   int offset_calc=frame_width*frame_height;

   printf("\n\nCalculated offset : %d\n\n", offset_calc);
   int offset_proc= (surface->surfaceList[0].planeParams.psize[0] - offset_calc);

   const uint8_t *Y_mapped=surface->surfaceList[i].mappedAddr.addr[0]; 

   if(Y_mapped != NULL)
   {
	char filename[200];
	sprintf(filename,"y_%dx%d.jpg",frame_width,frame_height);
	write_jpeg_file( filename, Y_mapped , frame_width, frame_height, 1, JCS_GRAYSCALE );
   }
   else
   {
	printf("\nY == NULL or UV != NULL\n");
   }

  }
 
  int unmap_result= NvBufSurfaceUnMap(surface,-1,-1);

  exit(0); 

#endif
  /*
   * Output KITTI labels with tracking ID if configured to do so.
   */
  write_kitti_track_output(appCtx, batch_meta);

  //send_kitti_track_output(appCtx, batch_meta);

  if (appCtx->primary_bbox_generated_cb)
    appCtx->primary_bbox_generated_cb (appCtx, buf, batch_meta, index);
  return GST_PAD_PROBE_OK;
}

In NvBufSurface documentation, I could find “STRUCTURE_PADDING” and “reserved” data members. Also, in NvBufSurfaceMappedAddr struct, I could find “eglImage”. Should I use these in a particular way to extract the frame buffer ? If so, how to use them ?

Kindly help me out.

Thanks.

Hi,
For accessing the frames, please refer to gst-dsexample in

deepstream_sdk_v4.0_jetson\sources\gst-plugins\gst-dsexample

You can follow the README to enable it in running deepstream-app.

Also below is a sample code of extracting the objects and saving to jpeg files.
https://devtalk.nvidia.com/default/topic/1061422/deepstream-sdk/how-to-crop-the-image-and-save/post/5375174/#5375174

Hi DaneLLL,

Thanks for the pointer. I have managed to run the plugin by enabling it with deepstream-app. In the “get_converted_mat” function I am trying to dump the output BGR frame by using imwrite function of opencv. However, when I dump the output, it does not seem to be right. I am attaching the input and output for the experiment that I conducted. Please find the code below:

static GstFlowReturn
get_converted_mat (GstDsExample * dsexample, NvBufSurface *input_buf, gint idx,
    NvOSD_RectParams * crop_rect_params, gdouble & ratio, gint input_width,
    gint input_height)
{
  printf("\ndsexample: Entered get_converted_mat\n");

  NvBufSurfTransform_Error err;
  NvBufSurfTransformConfigParams transform_config_params;
  NvBufSurfTransformParams transform_params;
  NvBufSurfTransformRect src_rect;
  NvBufSurfTransformRect dst_rect;
  NvBufSurface ip_surf;
  cv::Mat in_mat;
  ip_surf = *input_buf;

  ip_surf.numFilled = ip_surf.batchSize = 1;
  ip_surf.surfaceList = &(input_buf->surfaceList[idx]);

  gint src_left = GST_ROUND_UP_2(crop_rect_params->left);
  gint src_top = GST_ROUND_UP_2(crop_rect_params->top);
  gint src_width = GST_ROUND_DOWN_2(crop_rect_params->width);
  gint src_height = GST_ROUND_DOWN_2(crop_rect_params->height);

  // Maintain aspect ratio
  double hdest = dsexample->processing_width * src_height / (double) src_width;
  double wdest = dsexample->processing_height * src_width / (double) src_height;
  guint dest_width, dest_height;

  if (hdest <= dsexample->processing_height) {
    dest_width = dsexample->processing_width;
    dest_height = hdest;
  } else {
    dest_width = wdest;
    dest_height = dsexample->processing_height;
  }

  // Configure transform session parameters for the transformation
  transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
  transform_config_params.gpu_id = dsexample->gpu_id;
  transform_config_params.cuda_stream = dsexample->cuda_stream;

  // Set the transform session parameters for the conversions executed in this
  // thread.
  err = NvBufSurfTransformSetSessionParams (&transform_config_params);
  if (err != NvBufSurfTransformError_Success) {
    GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
        ("NvBufSurfTransformSetSessionParams failed with error %d", err), (NULL));
    goto error;
  }

  // Calculate scaling ratio while maintaining aspect ratio
  ratio = MIN (1.0 * dest_width/ src_width, 1.0 * dest_height / src_height);

  if ((crop_rect_params->width == 0) || (crop_rect_params->height == 0)) {
    GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
        ("%s:crop_rect_params dimensions are zero",__func__), (NULL));
    goto error;
  }

#ifdef __aarch64__
  if (ratio <= 1.0 / 16 || ratio >= 16.0) {
    // Currently cannot scale by ratio > 16 or < 1/16 for Jetson
    goto error;
  }
#endif
  // Set the transform ROIs for source and destination
  src_rect = {(guint)src_top, (guint)src_left, (guint)src_width, (guint)src_height};
  dst_rect = {0, 0, (guint)dest_width, (guint)dest_height};

  // Set the transform parameters
  transform_params.src_rect = &src_rect;
  transform_params.dst_rect = &dst_rect;
  transform_params.transform_flag =
    NVBUFSURF_TRANSFORM_FILTER | NVBUFSURF_TRANSFORM_CROP_SRC |
      NVBUFSURF_TRANSFORM_CROP_DST;
  transform_params.transform_filter = NvBufSurfTransformInter_Default;

  //Memset the memory
  NvBufSurfaceMemSet (dsexample->inter_buf, 0, 0, 0);

  GST_DEBUG_OBJECT (dsexample, "Scaling and converting input buffer\n");

  // Transformation scaling+format conversion if any.
  err = NvBufSurfTransform (&ip_surf, dsexample->inter_buf, &transform_params);
  if (err != NvBufSurfTransformError_Success) {
    GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
        ("NvBufSurfTransform failed with error %d while converting buffer", err),
        (NULL));
    goto error;
  }


  // Map the buffer so that it can be accessed by CPU
  if (NvBufSurfaceMap (dsexample->inter_buf, 0, 0, NVBUF_MAP_READ) != 0){
    goto error;
  }

  // Cache the mapped data for CPU access
  NvBufSurfaceSyncForCpu (dsexample->inter_buf, 0, 0);

  printf("Processing Height : %d \n",  dsexample->processing_height);
  printf("Processing Width : %d \n",  dsexample->processing_width);
  printf("Pitch : %d \n",  dsexample->inter_buf->surfaceList[0].pitch);
  printf("isContiguous : %d\n",dsexample->inter_buf->isContiguous);


  // Use openCV to remove padding and convert RGBA to BGR. Can be skipped if
  // algorithm can handle padded RGBA data.

  in_mat =
      cv::Mat (dsexample->processing_height, dsexample->processing_width,
      CV_8UC4, dsexample->inter_buf->surfaceList[0].mappedAddr.addr[0],
      dsexample->inter_buf->surfaceList[0].pitch);


  cv::cvtColor (in_mat, *dsexample->cvmat, CV_RGBA2BGR);
  cv::imwrite("test.jpg",*dsexample->cvmat);


  if (NvBufSurfaceUnMap (dsexample->inter_buf, 0, 0)){
    goto error;
  }

#ifdef __aarch64__
  // To use the converted buffer in CUDA, create an EGLImage and then use
  // CUDA-EGL interop APIs
  if (USE_EGLIMAGE) {
    if (NvBufSurfaceMapEglImage (dsexample->inter_buf, 0) !=0 ) {
      goto error;
    }

    // dsexample->inter_buf->surfaceList[0].mappedAddr.eglImage
    // Use interop APIs cuGraphicsEGLRegisterImage and
    // cuGraphicsResourceGetMappedEglFrame to access the buffer in CUDA

    // Destroy the EGLImage
    NvBufSurfaceUnMapEglImage (dsexample->inter_buf, 0);
  }
#endif

  /* We will first convert only the Region of Interest (the entire frame or the
   * object bounding box) to RGB and then scale the converted RGB frame to
   * processing resolution. */
  return GST_FLOW_OK;

error:
  return GST_FLOW_ERROR;
}

Kindly help me out.

Thanks.
debug.zip (7.43 MB)

Hi,
Please set width and height to be same as your source:

processing-width=1280
processing-height=720

Hi DaneLLL,

I changed the parameters as suggested by you. Still the output does not seem to be write. I am attaching the log containing the pixel values as well as the output image along with this post. Changes made:

[ds-example]
enable=1
processing-width=1280
processing-height=720
full-frame=0
unique-id=15
gpu-id=0

Code for dumping values :

void parse_ychannel_row_major(uint32_t width, uint32_t height,uint8_t *Y,uint32_t Y_stride)
{
	printf("\nPrinting parse_ychannel_row_major\n\n");
	int x,y;
	for(y=0; y< height; ++y)
	{
		uint8_t *y_row_ptr=Y+y*Y_stride;

		for(x=0;x<width;++x)
		{
			uint8_t *data_ptr=y_row_ptr+x;
			if(data_ptr != NULL)
			{
				uint8_t data = *(data_ptr);
				printf("Y Pixel Value : %d  x : %d , y : %d \n",data,x,y);
			}
			else
			{
				printf("\n\nparse_ychannel_row_major : Null pointer detected\n\n");
			}
		}
	}
}
...
  printf("Processing Height : %d \n",  dsexample->processing_height);
  printf("Processing Width : %d \n",  dsexample->processing_width);
  printf("Pitch : %d \n",  dsexample->inter_buf->surfaceList[0].pitch);
  printf("isContiguous : %d\n",dsexample->inter_buf->isContiguous);

  parse_ychannel_row_major(dsexample->inter_buf->surfaceList[0].pitch,
			 dsexample->processing_height,
			(uint8_t *) dsexample->inter_buf->surfaceList[0].mappedAddr.addr[0],
			dsexample->inter_buf->surfaceList[0].pitch);

  // Use openCV to remove padding and convert RGBA to BGR. Can be skipped if
  // algorithm can handle padded RGBA data.

  in_mat =
      cv::Mat (dsexample->processing_height, dsexample->processing_width,
      CV_8UC4, dsexample->inter_buf->surfaceList[0].mappedAddr.addr[0],
      dsexample->inter_buf->surfaceList[0].pitch);

  cv::cvtColor (in_mat, *dsexample->cvmat, CV_RGBA2BGR);
  cv::imwrite("test.jpg",*dsexample->cvmat);
  cv::imwrite("test_in_mat.jpg",in_mat);
...

Thanks
debug_proc_1280x720.zip (23.8 MB)

Hi,
Please also configure

full-frame=1

Hi DaneLLL,

I am able to get the dumps now sometimes but not all the time. What could be the reason behind this variability? Also, when I try to replicate the logic in “tracking_done_prob” function defined in “deepstream-app.c”, it does not seem to work and crashes. Below is the code:

/**
 * Buffer probe function after tracker.
 */
static GstPadProbeReturn
tracking_done_buf_prob (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
  NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
  guint index = bin->index;
  AppCtx *appCtx = bin->appCtx;
  GstBuffer *buf = (GstBuffer *) info->data;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  if (!batch_meta) {
    NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
    return GST_PAD_PROBE_OK;
  }

  GstMapInfo in_map_info;
  NvBufSurface *surface = NULL;

  memset (&in_map_info, 0, sizeof (in_map_info));
  if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
    g_print ("Error: Failed to map gst buffer\n");
   gst_buffer_unmap (buf, &in_map_info);
   return -1;
  }

  cudaError_t cuda_err;

  NvBufSurfTransformRect src_rect, dst_rect;
  surface = (NvBufSurface *) in_map_info.data;  
  
  int batch_size= surface->batchSize;
  printf("\nBatch Size : %d\n",batch_size );

  src_rect.top   = 0;
  src_rect.left  = 0;
  src_rect.width = (guint) surface->surfaceList[0].width;
  src_rect.height= (guint) surface->surfaceList[0].height;

  dst_rect.top   = 0;
  dst_rect.left  = 0;
  dst_rect.width = (guint) surface->surfaceList[0].width;
  dst_rect.height= (guint) surface->surfaceList[0].height;

  NvBufSurfTransformParams nvbufsurface_params;
  nvbufsurface_params.src_rect = &src_rect;
  nvbufsurface_params.dst_rect = &dst_rect;
  nvbufsurface_params.transform_flag =  NVBUFSURF_TRANSFORM_CROP_SRC |	NVBUFSURF_TRANSFORM_CROP_DST;
  nvbufsurface_params.transform_filter = NvBufSurfTransformInter_Default;

NvBufSurface *dst_surface;
  NvBufSurfaceCreateParams nvbufsurface_create_params;

  if (dst_surface)
    NvBufSurfaceDestroy (dst_surface);
  dst_surface = NULL;

  /* An intermediate buffer for NV12/RGBA to BGR conversion  will be
   * required. Can be skipped if custom algorithm can work directly on NV12/RGBA. */
  nvbufsurface_create_params.gpuId  = surface->gpuId;
  nvbufsurface_create_params.width  = (gint) surface->surfaceList[0].width;
  nvbufsurface_create_params.height = (gint) surface->surfaceList[0].height;
  nvbufsurface_create_params.size = 0;
  nvbufsurface_create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
  nvbufsurface_create_params.layout = NVBUF_LAYOUT_PITCH;
  nvbufsurface_create_params.memType = NVBUF_MEM_DEFAULT;

  cuda_err = cudaSetDevice (surface->gpuId);
  printf("\cudaSetDevice : %d\n",cuda_err);

  cudaStream_t cuda_stream;

  cuda_err=cudaStreamCreate (&cuda_stream);

  printf("\cudaStreamCreate : %d\n",cuda_err);

  int create_result = NvBufSurfaceCreate(&dst_surface,batch_size,&nvbufsurface_create_params);	
  
  printf("\ncreate_result : %d\n", create_result);

  NvBufSurfTransformConfigParams transform_config_params;
  NvBufSurfTransform_Error err;

  transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
  transform_config_params.gpu_id = surface->gpuId;
  transform_config_params.cuda_stream = cuda_stream;

  err = NvBufSurfTransformSetSessionParams (&transform_config_params);
  printf("\nError code returned by set session params : %d\n",err);

  NvBufSurfaceMemSet (dst_surface, 0, 0, 0);

  err = NvBufSurfTransform (surface, dst_surface, &nvbufsurface_params);
  if (err != NvBufSurfTransformError_Success) {
	  g_print ("NvBufSurfTransform failed with error %d while converting buffer\n", err);
  }

  printf("\nNvBufSurfTransform_Error : %d\n",err );

  NvBufSurfaceMap (dst_surface, 0, 0, NVBUF_MAP_READ);

  NvBufSurfaceSyncForCpu (dst_surface, 0, 0);

  void *host_rgb_buf;
  int rgb_bytes_per_pixel=3;

  cuda_err = cudaMallocHost (&host_rgb_buf,nvbufsurface_create_params.width * nvbufsurface_create_params.height * rgb_bytes_per_pixel );

  cv::Mat bgr_frame = cv::Mat ( nvbufsurface_create_params.height, nvbufsurface_create_params.width,
      CV_8UC3, host_rgb_buf,
      nvbufsurface_create_params.width * rgb_bytes_per_pixel);

  cv::Mat in_mat =
      cv::Mat (nvbufsurface_create_params.height, nvbufsurface_create_params.width,
      CV_8UC4, dst_surface->surfaceList[0].mappedAddr.addr[0],
      dst_surface->surfaceList[0].pitch);

  cv::cvtColor (in_mat, bgr_frame, CV_RGBA2BGR);
  char string_to_print[200];
  sprintf(string_to_print,"test_%lu.jpg", (int)time(NULL));

  cv::imwrite(string_to_print,bgr_frame);
  cv::imwrite("test_in_mat.jpg",in_mat);

  NvBufSurfaceUnMap (dst_surface, 0, 0);

  /*
   * Output KITTI labels with tracking ID if configured to do so.
   */
  write_kitti_track_output(appCtx, batch_meta);

if (appCtx->primary_bbox_generated_cb)
    appCtx->primary_bbox_generated_cb (appCtx, buf, batch_meta, index);
  return GST_PAD_PROBE_OK;
}

All the error prints return 0 (success)

The program crashes and gives segmentation fault at the imwrite calls. Kindly let me know if you can sight any errors in this code.

Thanks.

Hi,
for more information, does it crash in first or second imwrite()?

Hi,
Below code looks not right.

NvBufSurface *dst_surface;
NvBufSurfaceCreateParams nvbufsurface_create_params;

if (dst_surface)
NvBufSurfaceDestroy (dst_surface);
dst_surface = NULL;

It would be better to destroy the surface and cuda memory at the end of tracking_done_prob

Hi DaneLLL,

The crash is observed in the first imwrite call. I will try your suggestion and report.

Thanks

Hi DaneLLL,

Still does not work.

Hi,
Please share changes in deepstream_sdk_v4.0_jetson/sources/apps/sample_apps/deepstream-app/Makefile
So that we can build and run the app to reproduce the failure.

Hi DaneLLL,

Please find the changes below:

APP:= deepstream-app

#TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -)
TARGET_DEVICE = $(shell g++ -dumpmachine | cut -f1 -d -)

NVDS_VERSION:=4.0

LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(NVDS_VERSION)/lib/

ifeq ($(TARGET_DEVICE),aarch64)
  CFLAGS:= -DPLATFORM_TEGRA
endif

CUDA_VER:=10.0

SRCS:= $(wildcard *.c)
SRCS+= $(wildcard ../../apps-common/src/*.c)

INCS:= $(wildcard *.h)

PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11 opencv

OBJS:= $(SRCS:.c=.o)
CC := g++
  
CFLAGS+= -I../../apps-common/includes -I../../../includes -I/home/edgetensor/Documents/external/curl/include -I/usr/src/nvidia/tegra_multimedia_api/include/ -I/usr/local/cuda-10.0/targets/aarch64-linux/include/  /usr/local/cuda-$(CUDA_VER)/include -I/home/edgetensor/Documents/external/libjpeg/ -I/home/edgetensor/Documents/external/libjpeg/build  -DDS_VERSION_MINOR=0 -DDS_VERSION_MAJOR=4 -fpermissive -Wnarrowing 

LIBS+= -L$(LIB_INSTALL_DIR) -L/home/edgetensor/Documents/external/curl/build/lib -L/usr/local/cuda-10.0/targets/aarch64-linux/lib -L/home/edgetensor/Documents/external/libjpeg/build  -L/usr/local/cuda-$(CUDA_VER)/lib64/  -lcudart -ljpeg  -lnppicc -lnvdsgst_meta -lnvbufsurface -lnvbufsurftransform -lnvds_meta -lnvdsgst_helper -lnvds_utils -lm -lcurl \
       -lgstrtspserver-1.0 -Wl,-rpath,$(LIB_INSTALL_DIR)

CFLAGS+= `pkg-config --cflags $(PKGS)`

LIBS+= `pkg-config --libs $(PKGS)`

all: $(APP)

%.o: %.c $(INCS) Makefile
	$(CC) -c -o $@ $(CFLAGS) $<

$(APP): $(OBJS) Makefile
	$(CC) -o $(APP) $(OBJS) $(LIBS)

clean:
	rm -rf $(OBJS) $(APP)

Thanks.

Hi DaneLLL,

I managed to get rid of the segmentation fault by replacing the imwrite calls by cvSaveImage. Kindly find the code below:

...
  cv::cvtColor (in_mat, *bgr_frame, CV_RGBA2BGR);
  CvMat cvMat = *bgr_frame;
  cvSaveImage( "saveimage_dump.jpg", &cvMat );
...

I am attaching a sample dump so that you can relate to the issue better. Kindly point me out how to rectify this.

Thanks.


Hi,
Adding the following code to tracking_done_buf_prob() works fine. For your reference.

#if 1
  static int dump = 0;
  if (dump < 150) {
    GstMapInfo in_map_info;
    NvBufSurface *surface = NULL;

    memset (&in_map_info, 0, sizeof (in_map_info));
    if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
      g_print ("Error: Failed to map gst buffer\n");
      gst_buffer_unmap (buf, &in_map_info);
      return GST_PAD_PROBE_OK;
    }
    cudaError_t cuda_err;

    NvBufSurfTransformRect src_rect, dst_rect;
    surface = (NvBufSurface *) in_map_info.data;  
  
    int batch_size= surface->batchSize;
    printf("\nBatch Size : %d, resolution : %dx%d \n",batch_size,
        surface->surfaceList[0].width, surface->surfaceList[0].height);

    src_rect.top   = 0;
    src_rect.left  = 0;
    src_rect.width = (guint) surface->surfaceList[0].width;
    src_rect.height= (guint) surface->surfaceList[0].height;

    dst_rect.top   = 0;
    dst_rect.left  = 0;
    dst_rect.width = (guint) surface->surfaceList[0].width;
    dst_rect.height= (guint) surface->surfaceList[0].height;

    NvBufSurfTransformParams nvbufsurface_params;
    nvbufsurface_params.src_rect = &src_rect;
    nvbufsurface_params.dst_rect = &dst_rect;
    nvbufsurface_params.transform_flag =  NVBUFSURF_TRANSFORM_CROP_SRC | NVBUFSURF_TRANSFORM_CROP_DST;
    nvbufsurface_params.transform_filter = NvBufSurfTransformInter_Default;
  
    NvBufSurface *dst_surface = NULL;
    NvBufSurfaceCreateParams nvbufsurface_create_params;

    /* An intermediate buffer for NV12/RGBA to BGR conversion  will be
     * required. Can be skipped if custom algorithm can work directly on NV12/RGBA. */
    nvbufsurface_create_params.gpuId  = surface->gpuId;
    nvbufsurface_create_params.width  = (gint) surface->surfaceList[0].width;
    nvbufsurface_create_params.height = (gint) surface->surfaceList[0].height;
    nvbufsurface_create_params.size = 0;
    nvbufsurface_create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
    nvbufsurface_create_params.layout = NVBUF_LAYOUT_PITCH;
    nvbufsurface_create_params.memType = NVBUF_MEM_DEFAULT;

    cuda_err = cudaSetDevice (surface->gpuId);

    cudaStream_t cuda_stream;

    cuda_err=cudaStreamCreate (&cuda_stream);

    int create_result = NvBufSurfaceCreate(&dst_surface,batch_size,&nvbufsurface_create_params);	

    NvBufSurfTransformConfigParams transform_config_params;
    NvBufSurfTransform_Error err;

    transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
    transform_config_params.gpu_id = surface->gpuId;
    transform_config_params.cuda_stream = cuda_stream;
    err = NvBufSurfTransformSetSessionParams (&transform_config_params);

    NvBufSurfaceMemSet (dst_surface, 0, 0, 0);
    err = NvBufSurfTransform (surface, dst_surface, &nvbufsurface_params);
    if (err != NvBufSurfTransformError_Success) {
  	  g_print ("NvBufSurfTransform failed with error %d while converting buffer\n", err);
    }
    NvBufSurfaceMap (dst_surface, 0, 0, NVBUF_MAP_READ);
    NvBufSurfaceSyncForCpu (dst_surface, 0, 0);

    cv::Mat bgr_frame = cv::Mat (cv::Size(nvbufsurface_create_params.width, nvbufsurface_create_params.height), CV_8UC3);

    cv::Mat in_mat =
        cv::Mat (nvbufsurface_create_params.height, nvbufsurface_create_params.width,
        CV_8UC4, dst_surface->surfaceList[0].mappedAddr.addr[0],
        dst_surface->surfaceList[0].pitch);

    cv::cvtColor (in_mat, bgr_frame, CV_RGBA2BGR);

    char filename[64];
    snprintf(filename, 64, "/tmp/image%03d.jpg", dump);
    cv::imwrite(filename,bgr_frame);
    dump ++;

    NvBufSurfaceUnMap (dst_surface, 0, 0);
    NvBufSurfaceDestroy (dst_surface);
    cudaStreamDestroy (cuda_stream);
    gst_buffer_unmap (buf, &in_map_info);
  }
#endif
1 Like

Thanks DaneLLL. This seems to work. However, could you please explain conceptually what went wrong earlier?