RTSP camera access frame issue

[neophyte1]

Hi Guys,

I am trying to run deepstream-app using RTSP. I am trying to access the frame in the tracking probe function in deepstream-app. I referred an earlier post :

https://devtalk.nvidia.com/default/topic/1058086/deepstream-sdk/how-to-run-rtp-camera-in-deepstream-on-nano/post/5374027/#5374027

From this post, I could understand that the color format of the frame is NvBufferColorFormat_NV12_709_ER which is YUV420 multi-planar.However, I am unable to understand the dataSize, pitch, width and height values got from NvBufSurfaceParams.

/**
 * Buffer probe function after tracker.
 */
static GstPadProbeReturn
tracking_done_buf_prob (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
  NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
  guint index = bin->index;
  AppCtx *appCtx = bin->appCtx;
  GstBuffer *buf = (GstBuffer *) info->data;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  if (!batch_meta) {
    NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
    return GST_PAD_PROBE_OK;
  }

  GstMapInfo in_map_info;
  NvBufSurface *surface = NULL;

  memset (&in_map_info, 0, sizeof (in_map_info));
  if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
    g_print ("Error: Failed to map gst buffer\n");
  }
  surface = (NvBufSurface *) in_map_info.data;  

  int batch_size= surface->batchSize;

  for(int i=0; i<batch_size; ++i)
  {
	uint32_t data_size =  surface->surfaceList[i].dataSize;
	uint32_t pitch =  surface->surfaceList[i].pitch;
	uint32_t width =  surface->surfaceList[i].width;
	uint32_t height =  surface->surfaceList[i].height;

	void *dataPtr = surface->surfaceList[i].dataPtr;
	
	printf("Size of the frame buffer : %d\n",data_size);
	printf("Pitch of the frame buffer : %d\n",pitch);
	printf("width of the frame buffer : %d\n",width);
	printf("height of the frame buffer : %d\n",height);

	NvBufSurfaceColorFormat color_format= surface->surfaceList[i].colorFormat;

        if (color_format == NVBUF_COLOR_FORMAT_NV12)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_ER \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709_ER \n");
  }

  /*
   * Output KITTI labels with tracking ID if configured to do so.
   */
  write_kitti_track_output(appCtx, batch_meta);

  if (appCtx->primary_bbox_generated_cb)
    appCtx->primary_bbox_generated_cb (appCtx, buf, batch_meta, index);
  return GST_PAD_PROBE_OK;
}

Following is the output :

Size of the frame buffer : 1572864
Pitch of the frame buffer : 1280
width of the frame buffer : 1280
height of the frame buffer : 720

I am unable to comprehend what is going on. Kindly let me know where I am going wrong or is there any information which I am missing out on.

Thanks.

[neophyte1]

Hi Guys,

I tried to dump other information to get more insights. Please find the code below and the dumps:

Code :

/**
 * Buffer probe function after tracker.
 */
static GstPadProbeReturn
tracking_done_buf_prob (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
  NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
  guint index = bin->index;
  AppCtx *appCtx = bin->appCtx;
  GstBuffer *buf = (GstBuffer *) info->data;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  if (!batch_meta) {
    NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
    return GST_PAD_PROBE_OK;
  }

  GstMapInfo in_map_info;
  NvBufSurface *surface = NULL;

  memset (&in_map_info, 0, sizeof (in_map_info));
  if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
    g_print ("Error: Failed to map gst buffer\n");
  }
  surface = (NvBufSurface *) in_map_info.data;  

  int batch_size= surface->batchSize;

  for(int i=0; i<batch_size; ++i)
  {
	uint32_t data_size =  surface->surfaceList[i].dataSize;
	uint32_t pitch =  surface->surfaceList[i].pitch;
	uint32_t width =  surface->surfaceList[i].width;
	uint32_t height =  surface->surfaceList[i].height;
	NvBufSurfaceLayout layout = surface->surfaceList[i].layout;
	NvBufSurfacePlaneParams plane_params = surface->surfaceList[i].planeParams;
 	void *dataPtr = surface->surfaceList[i].dataPtr;
	
	uint32_t num_planes=plane_params.num_planes;
	
	printf("\nNumber of Planes : %d\n\n", num_planes);

	for(int i=0 ; i < num_planes; ++i)
	{
	   uint32_t width=plane_params.width[i];
	   uint32_t height=plane_params.height[i];
	   uint32_t pitch=plane_params.pitch[i]; 
           uint32_t offset=plane_params.offset[i];
           uint32_t psize=plane_params.psize[i];
           uint32_t bytes_per_pix=plane_params.bytesPerPix[i];
		
	   printf("width of the plane %d : %d\n\n",i,width);
	   printf("height of the plane %d : %d\n\n",i,height);
	   printf("pitch of the plane  %d : %d\n\n",i,pitch);
	   printf("offset of the plane  %d : %d\n\n",i,offset);
	   printf("psize of the plane  %d : %d\n\n",i,psize);
	   printf("bytes_per_pix of the plane  %d : %d\n\n\n\n",i,bytes_per_pix);
	   
	}


	printf("Size of the frame buffer : %d\n\n",data_size);
	printf("Pitch of the frame buffer : %d\n\n",pitch);
	printf("width of the frame buffer : %d\n\n",width);
	printf("height of the frame buffer : %d\n\n",height);

	NvBufSurfaceColorFormat color_format= surface->surfaceList[i].colorFormat;

        if (color_format == NVBUF_COLOR_FORMAT_NV12)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_ER \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709_ER \n");
  }

  /*
   * Output KITTI labels with tracking ID if configured to do so.
   */
  write_kitti_track_output(appCtx, batch_meta);

  if (appCtx->primary_bbox_generated_cb)
    appCtx->primary_bbox_generated_cb (appCtx, buf, batch_meta, index);
  return GST_PAD_PROBE_OK;
}

Dumps:

Number of Planes : 2

width of the plane 0 : 1280
height of the plane 0 : 720
pitch of the plane  0 : 1280
offset of the plane  0 : 0
psize of the plane  0 : 1048576
bytes_per_pix of the plane  0 : 1

width of the plane 1 : 640
height of the plane 1 : 360
pitch of the plane  1 : 1280
offset of the plane  1 : 1048576
psize of the plane  1 : 524288
bytes_per_pix of the plane  1 : 2

Size of the frame buffer : 1572864
Pitch of the frame buffer : 1280
width of the frame buffer : 1280
height of the frame buffer : 720

I am still unable to understand the values of psize and dataSize.

Kindly help me out.

Thanks.

[neophyte1]

Hi Guys,

I get segmentation fault midway when I just try to access the Y channel buffer. Also, I am unable to understand the dumps I get by printing the parameters.

Code:

void parse_ychannel_row_major(uint32_t width, uint32_t height,const uint8_t *Y,uint32_t Y_stride)
{
	int x,y;
	for(y=0; y< height; ++y)
	{
		const uint8_t *y_row_ptr=Y+y*Y_stride;

		for(x=0;x<width;++x)
		{
			uint8_t data = *(y_row_ptr+x);
			printf("Y Pixel Value : %d\n",data);
			printf("x : %d , y : %d \n",x,y);
		}

	}

}

static GstPadProbeReturn
tracking_done_buf_prob (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
  NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
  guint index = bin->index;
  AppCtx *appCtx = bin->appCtx;
  GstBuffer *buf = (GstBuffer *) info->data;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  if (!batch_meta) {
    NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
    return GST_PAD_PROBE_OK;
  }

  GstMapInfo in_map_info;
  NvBufSurface *surface = NULL;

  memset (&in_map_info, 0, sizeof (in_map_info));
  if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
    g_print ("Error: Failed to map gst buffer\n");
  }

#if 1

  surface = (NvBufSurface *) in_map_info.data;  

  int batch_size= surface->batchSize;

  printf("Batch Size : %d",batch_size );

for(int i=0; i<batch_size; ++i)
  {
	uint32_t data_size =  surface->surfaceList[i].dataSize;
	uint32_t pitch =  surface->surfaceList[i].pitch;
	uint32_t width =  surface->surfaceList[i].width;
	uint32_t height =  surface->surfaceList[i].height;
	NvBufSurfaceLayout layout = surface->surfaceList[i].layout;
	NvBufSurfacePlaneParams plane_params = surface->surfaceList[i].planeParams;
 	void *dataPtr = surface->surfaceList[i].dataPtr;
	uint8_t *data = dataPtr;
	
 	//printf("\nData at first index buffer %d : %d",i,*data);

	uint32_t num_planes=plane_params.num_planes;
	
	printf("\nNumber of Planes : %d\n\n", num_planes);

	for(int i=0 ; i < num_planes; ++i)
	{
	   uint32_t width=plane_params.width[i];
	   uint32_t height=plane_params.height[i];
	   uint32_t pitch=plane_params.pitch[i]; 
           uint32_t offset=plane_params.offset[i];
           uint32_t psize=plane_params.psize[i];
           uint32_t bytes_per_pix=plane_params.bytesPerPix[i];
		
	   printf("width of the plane %d : %d\n\n",i,width);
	   printf("height of the plane %d : %d\n\n",i,height);
	   printf("pitch of the plane  %d : %d\n\n",i,pitch);
	   printf("offset of the plane  %d : %d\n\n",i,offset);
	   printf("psize of the plane  %d : %d\n\n",i,psize);
	   printf("bytes_per_pix of the plane  %d : %d\n\n\n\n",i,bytes_per_pix);	   
	}

	printf("Size of the frame buffer : %d\n\n",data_size);
	printf("Pitch of the frame buffer : %d\n\n",pitch);
	printf("width of the frame buffer : %d\n\n",width);
	printf("height of the frame buffer : %d\n\n",height);

	NvBufSurfaceColorFormat color_format= surface->surfaceList[i].colorFormat;

        if (color_format == NVBUF_COLOR_FORMAT_NV12)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_ER \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709_ER \n");
  }

uint32_t frame_width=surface->surfaceList[0].planeParams.width[0];
   uint32_t frame_height=surface->surfaceList[0].planeParams.height[0];
   uint32_t Y_stride=surface->surfaceList[0].planeParams.pitch[0];
   uint32_t UV_stride=surface->surfaceList[0].planeParams.pitch[1];

   printf("\n\nframe_width : %d\n\n", frame_width);
   printf("\n\nframe_height : %d\n\n", frame_height);
   printf("\n\nY_stride : %d\n\n", Y_stride);
   printf("\n\nUV_stride : %d\n\n", UV_stride);

int offset_calc=frame_width*frame_height;

   //Method 1
   const uint8_t *Y=surface->surfaceList[0].dataPtr;

   // Method 2 
   //const uint8_t *Y=surface->surfaceList[0].dataPtr + (surface->surfaceList[0].planeParams.psize[0] - offset_calc);
   
   parse_ychannel_row_major(frame_width, frame_height,Y,Y_stride);

  exit(0); 

#endif
  /*
   * Output KITTI labels with tracking ID if configured to do so.
   */
  write_kitti_track_output(appCtx, batch_meta);

  if (appCtx->primary_bbox_generated_cb)
    appCtx->primary_bbox_generated_cb (appCtx, buf, batch_meta, index);
  return GST_PAD_PROBE_OK;
}

Beginning of the log with the parameters :

Creating LL OSD context new
KLT Tracker Init
Batch Size : 1
Number of Planes : 2

width of the plane 0 : 1280
height of the plane 0 : 720
pitch of the plane  0 : 1280
offset of the plane  0 : 0
psize of the plane  0 : 1048576
bytes_per_pix of the plane  0 : 1

width of the plane 1 : 640
height of the plane 1 : 360
pitch of the plane  1 : 1280
offset of the plane  1 : 1048576
psize of the plane  1 : 524288
bytes_per_pix of the plane  1 : 2

Size of the frame buffer : 1572864
Pitch of the frame buffer : 1280
width of the frame buffer : 1280
height of the frame buffer : 720

color_format: NVBUF_COLOR_FORMAT_NV12_709_ER

frame_width : 1280
frame_height : 720
Y_stride : 1280
UV_stride : 1280

The end of the log looks like the following before it crashes:

Y Pixel Value : 0
x : 615 , y : 286 
Y Pixel Value : 0
x : 616 , y : 286 
Y Pixel Value : 0
x : 617 , y : 286 
Y Pixel Value : 0
x : 618 , y : 286 
Y Pixel Value : 0
x : 619 , y : 286 
Y Pixel Value : 0
x : 620 , y : 286 
Y Pixel Value : 0
x : 621 , y : 286 
Y Pixel Value : 0
x : 622 , y : 286 
Y Pixel Value : 0
x : 623 , y : 286 
Y Pixel Value : 0
x : 624 , y : 286 
Y Pixel Value : 0
x : 625 , y : 286 
Y Pixel Value : 0
x : 626 , y : 286 
Y Pixel Value : 0
x : 627 , y : 286 
Y Pixel Value : 0
x : 628 , y : 286 
Y Pixel Value : 0
x : 629 , y : 286 
Y Pixel Value : 0
x Segmentation fault (core dumped)

The psize of the plane is greater than the product of width and height. Is there anything wrong with the way I am accessing the frame? Is there a possible memory access conflict?

Kindly help me out.

Thanks.

[joek5n31]

Pitch of the plane @1280 should always be in relation to a 720x1280 resolution.
I see you have 640x360. Try 640x480 and pitch of the plane @640 or maybe 720 but not 1280.
Try to alter the height and width.

[neophyte1]

Hi Guys,

I realised I needed to map the frame buffer pointer to CPU address space using “NvBufSurfaceMap”. Doing so, I have eliminated the problem of segmentation fault. However, when I dump the Y channel buffer using libjpeg, I do not get the right output (image attached). Please find the code below:

int write_jpeg_file( char *filename, unsigned char* rgb_image , int width, int height, int bytes_per_pixel, J_COLOR_SPACE color_space )
{

	struct jpeg_compress_struct cinfo;
	struct jpeg_error_mgr jerr;
	
	JSAMPROW row_pointer[1];
	FILE *outfile = fopen( filename, "wb" );
	
	if ( !outfile )
	{
		printf("Error opening output jpeg file %s\n!", filename );
		return -1;
	}
	cinfo.err = jpeg_std_error( &jerr );
	jpeg_create_compress(&cinfo);
	jpeg_stdio_dest(&cinfo, outfile);

	cinfo.image_width = width;	
	cinfo.image_height = height;
	cinfo.input_components = bytes_per_pixel;
	cinfo.in_color_space = color_space; //JCS_RGB

	jpeg_set_defaults( &cinfo );

	jpeg_start_compress( &cinfo, TRUE );

	while( cinfo.next_scanline < cinfo.image_height )
	{
		row_pointer[0] = &rgb_image[ cinfo.next_scanline * cinfo.image_width *  cinfo.input_components];
		jpeg_write_scanlines( &cinfo, row_pointer, 1 );
	}

	jpeg_finish_compress( &cinfo );
	jpeg_destroy_compress( &cinfo );
	fclose( outfile );

	return 1;
}

/**
 * Buffer probe function after tracker.
 */
static GstPadProbeReturn
tracking_done_buf_prob (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
  NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
  guint index = bin->index;
  AppCtx *appCtx = bin->appCtx;
  GstBuffer *buf = (GstBuffer *) info->data;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  if (!batch_meta) {
    NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
    return GST_PAD_PROBE_OK;
  }

  GstMapInfo in_map_info;
  NvBufSurface *surface = NULL;

  memset (&in_map_info, 0, sizeof (in_map_info));
  if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
    g_print ("Error: Failed to map gst buffer\n");
  }

#if 1

  surface = (NvBufSurface *) in_map_info.data;  

  int batch_size= surface->batchSize;

  printf("Batch Size : %d",batch_size );

  int map_result = NvBufSurfaceMap(surface,-1,-1,NVBUF_MAP_READ);	   
  printf("\nMap Result : %d\n", map_result);

  int sync_result = NvBufSurfaceSyncForCpu(surface,-1,-1);
  printf("\nSync Result : %d\n", sync_result);

  for(int i=0; i<batch_size; ++i)
  {
	uint32_t data_size =  surface->surfaceList[i].dataSize;
	uint32_t pitch =  surface->surfaceList[i].pitch;
	uint32_t width =  surface->surfaceList[i].width;
	uint32_t height =  surface->surfaceList[i].height;
	NvBufSurfaceLayout layout = surface->surfaceList[i].layout;
	NvBufSurfacePlaneParams plane_params = surface->surfaceList[i].planeParams;
 	void *dataPtr = surface->surfaceList[i].dataPtr;
	printf("\nSize of the first location : %d\n", sizeof(dataPtr[0]));
	uint8_t* data = dataPtr;
	
	printf("\n\nData pointer : %p\n",data);

	uint32_t num_planes=plane_params.num_planes;
	
	printf("\nNumber of Planes : %d\n\n", num_planes);

	for(int i=0 ; i < num_planes; ++i)
	{
	   uint32_t width=plane_params.width[i];
	   uint32_t height=plane_params.height[i];
	   uint32_t pitch=plane_params.pitch[i]; 
           uint32_t offset=plane_params.offset[i];
           uint32_t psize=plane_params.psize[i];
           uint32_t bytes_per_pix=plane_params.bytesPerPix[i];
		
	   printf("width of the plane %d : %d\n\n",i,width);
	   printf("height of the plane %d : %d\n\n",i,height);
	   printf("pitch of the plane  %d : %d\n\n",i,pitch);
	   printf("offset of the plane  %d : %d\n\n",i,offset);
	   printf("psize of the plane  %d : %d\n\n",i,psize);
	   printf("bytes_per_pix of the plane  %d : %d\n\n\n\n",i,bytes_per_pix);	   
	}

	printf("Size of the frame buffer : %d\n\n",data_size);
	printf("Pitch of the frame buffer : %d\n\n",pitch);
	printf("width of the frame buffer : %d\n\n",width);
	printf("height of the frame buffer : %d\n\n",height);

	NvBufSurfaceColorFormat color_format= surface->surfaceList[i].colorFormat;

        if (color_format == NVBUF_COLOR_FORMAT_NV12)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_ER \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709 \n");
        else if (color_format == NVBUF_COLOR_FORMAT_NV12_709_ER)
           printf("color_format: NVBUF_COLOR_FORMAT_NV12_709_ER \n");

uint32_t frame_width=surface->surfaceList[i].planeParams.width[0];
   uint32_t frame_height=surface->surfaceList[i].planeParams.height[0];
   uint32_t Y_stride=surface->surfaceList[i].planeParams.pitch[0];
   uint32_t UV_stride=surface->surfaceList[i].planeParams.pitch[1];

   printf("\n\nframe_width : %d\n\n", frame_width);
   printf("\n\nframe_height : %d\n\n", frame_height);
   printf("\n\nY_stride : %d\n\n", Y_stride);
   printf("\n\nUV_stride : %d\n\n", UV_stride);

   uint8_t *RGB= malloc(3*frame_width*frame_height);
   uint32_t RGB_stride=frame_width*3;
   YCbCrType yuv_type=YCBCR_709;
 
   printf("\n\nData at first index of RGB : %d\n\n", *RGB);
   printf("\n\nRGB_stride : %d\n\n", RGB_stride);

   int offset_calc=frame_width*frame_height;

   printf("\n\nCalculated offset : %d\n\n", offset_calc);
   int offset_proc= (surface->surfaceList[0].planeParams.psize[0] - offset_calc);

   const uint8_t *Y_mapped=surface->surfaceList[i].mappedAddr.addr[0]; 

   if(Y_mapped != NULL)
   {
	char filename[200];
	sprintf(filename,"y_%dx%d.jpg",frame_width,frame_height);
	write_jpeg_file( filename, Y_mapped , frame_width, frame_height, 1, JCS_GRAYSCALE );
   }
   else
   {
	printf("\nY == NULL or UV != NULL\n");
   }

  }
 
  int unmap_result= NvBufSurfaceUnMap(surface,-1,-1);

  exit(0); 

#endif
  /*
   * Output KITTI labels with tracking ID if configured to do so.
   */
  write_kitti_track_output(appCtx, batch_meta);

  //send_kitti_track_output(appCtx, batch_meta);

  if (appCtx->primary_bbox_generated_cb)
    appCtx->primary_bbox_generated_cb (appCtx, buf, batch_meta, index);
  return GST_PAD_PROBE_OK;
}

In NvBufSurface documentation, I could find “STRUCTURE_PADDING” and “reserved” data members. Also, in NvBufSurfaceMappedAddr struct, I could find “eglImage”. Should I use these in a particular way to extract the frame buffer ? If so, how to use them ?

Kindly help me out.

Thanks.

[DaneLLL]

Hi,
For accessing the frames, please refer to gst-dsexample in

deepstream_sdk_v4.0_jetson\sources\gst-plugins\gst-dsexample

You can follow the README to enable it in running deepstream-app.

Also below is a sample code of extracting the objects and saving to jpeg files.
https://devtalk.nvidia.com/default/topic/1061422/deepstream-sdk/how-to-crop-the-image-and-save/post/5375174/#5375174

[neophyte1]

Hi DaneLLL,

Thanks for the pointer. I have managed to run the plugin by enabling it with deepstream-app. In the “get_converted_mat” function I am trying to dump the output BGR frame by using imwrite function of opencv. However, when I dump the output, it does not seem to be right. I am attaching the input and output for the experiment that I conducted. Please find the code below:

static GstFlowReturn
get_converted_mat (GstDsExample * dsexample, NvBufSurface *input_buf, gint idx,
    NvOSD_RectParams * crop_rect_params, gdouble & ratio, gint input_width,
    gint input_height)
{
  printf("\ndsexample: Entered get_converted_mat\n");

  NvBufSurfTransform_Error err;
  NvBufSurfTransformConfigParams transform_config_params;
  NvBufSurfTransformParams transform_params;
  NvBufSurfTransformRect src_rect;
  NvBufSurfTransformRect dst_rect;
  NvBufSurface ip_surf;
  cv::Mat in_mat;
  ip_surf = *input_buf;

  ip_surf.numFilled = ip_surf.batchSize = 1;
  ip_surf.surfaceList = &(input_buf->surfaceList[idx]);

  gint src_left = GST_ROUND_UP_2(crop_rect_params->left);
  gint src_top = GST_ROUND_UP_2(crop_rect_params->top);
  gint src_width = GST_ROUND_DOWN_2(crop_rect_params->width);
  gint src_height = GST_ROUND_DOWN_2(crop_rect_params->height);

  // Maintain aspect ratio
  double hdest = dsexample->processing_width * src_height / (double) src_width;
  double wdest = dsexample->processing_height * src_width / (double) src_height;
  guint dest_width, dest_height;

  if (hdest <= dsexample->processing_height) {
    dest_width = dsexample->processing_width;
    dest_height = hdest;
  } else {
    dest_width = wdest;
    dest_height = dsexample->processing_height;
  }

  // Configure transform session parameters for the transformation
  transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
  transform_config_params.gpu_id = dsexample->gpu_id;
  transform_config_params.cuda_stream = dsexample->cuda_stream;

  // Set the transform session parameters for the conversions executed in this
  // thread.
  err = NvBufSurfTransformSetSessionParams (&transform_config_params);
  if (err != NvBufSurfTransformError_Success) {
    GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
        ("NvBufSurfTransformSetSessionParams failed with error %d", err), (NULL));
    goto error;
  }

  // Calculate scaling ratio while maintaining aspect ratio
  ratio = MIN (1.0 * dest_width/ src_width, 1.0 * dest_height / src_height);

  if ((crop_rect_params->width == 0) || (crop_rect_params->height == 0)) {
    GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
        ("%s:crop_rect_params dimensions are zero",__func__), (NULL));
    goto error;
  }

#ifdef __aarch64__
  if (ratio <= 1.0 / 16 || ratio >= 16.0) {
    // Currently cannot scale by ratio > 16 or < 1/16 for Jetson
    goto error;
  }
#endif
  // Set the transform ROIs for source and destination
  src_rect = {(guint)src_top, (guint)src_left, (guint)src_width, (guint)src_height};
  dst_rect = {0, 0, (guint)dest_width, (guint)dest_height};

  // Set the transform parameters
  transform_params.src_rect = &src_rect;
  transform_params.dst_rect = &dst_rect;
  transform_params.transform_flag =
    NVBUFSURF_TRANSFORM_FILTER | NVBUFSURF_TRANSFORM_CROP_SRC |
      NVBUFSURF_TRANSFORM_CROP_DST;
  transform_params.transform_filter = NvBufSurfTransformInter_Default;

  //Memset the memory
  NvBufSurfaceMemSet (dsexample->inter_buf, 0, 0, 0);

  GST_DEBUG_OBJECT (dsexample, "Scaling and converting input buffer\n");

  // Transformation scaling+format conversion if any.
  err = NvBufSurfTransform (&ip_surf, dsexample->inter_buf, &transform_params);
  if (err != NvBufSurfTransformError_Success) {
    GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
        ("NvBufSurfTransform failed with error %d while converting buffer", err),
        (NULL));
    goto error;
  }


  // Map the buffer so that it can be accessed by CPU
  if (NvBufSurfaceMap (dsexample->inter_buf, 0, 0, NVBUF_MAP_READ) != 0){
    goto error;
  }

  // Cache the mapped data for CPU access
  NvBufSurfaceSyncForCpu (dsexample->inter_buf, 0, 0);

  printf("Processing Height : %d \n",  dsexample->processing_height);
  printf("Processing Width : %d \n",  dsexample->processing_width);
  printf("Pitch : %d \n",  dsexample->inter_buf->surfaceList[0].pitch);
  printf("isContiguous : %d\n",dsexample->inter_buf->isContiguous);


  // Use openCV to remove padding and convert RGBA to BGR. Can be skipped if
  // algorithm can handle padded RGBA data.

  in_mat =
      cv::Mat (dsexample->processing_height, dsexample->processing_width,
      CV_8UC4, dsexample->inter_buf->surfaceList[0].mappedAddr.addr[0],
      dsexample->inter_buf->surfaceList[0].pitch);


  cv::cvtColor (in_mat, *dsexample->cvmat, CV_RGBA2BGR);
  cv::imwrite("test.jpg",*dsexample->cvmat);


  if (NvBufSurfaceUnMap (dsexample->inter_buf, 0, 0)){
    goto error;
  }

#ifdef __aarch64__
  // To use the converted buffer in CUDA, create an EGLImage and then use
  // CUDA-EGL interop APIs
  if (USE_EGLIMAGE) {
    if (NvBufSurfaceMapEglImage (dsexample->inter_buf, 0) !=0 ) {
      goto error;
    }

    // dsexample->inter_buf->surfaceList[0].mappedAddr.eglImage
    // Use interop APIs cuGraphicsEGLRegisterImage and
    // cuGraphicsResourceGetMappedEglFrame to access the buffer in CUDA

    // Destroy the EGLImage
    NvBufSurfaceUnMapEglImage (dsexample->inter_buf, 0);
  }
#endif

  /* We will first convert only the Region of Interest (the entire frame or the
   * object bounding box) to RGB and then scale the converted RGB frame to
   * processing resolution. */
  return GST_FLOW_OK;

error:
  return GST_FLOW_ERROR;
}

Kindly help me out.

Thanks.
debug.zip (7.43 MB)

[DaneLLL]

Hi,
Please set width and height to be same as your source:

processing-width=1280
processing-height=720

[neophyte1]

Hi DaneLLL,

I changed the parameters as suggested by you. Still the output does not seem to be write. I am attaching the log containing the pixel values as well as the output image along with this post. Changes made:

[ds-example]
enable=1
processing-width=1280
processing-height=720
full-frame=0
unique-id=15
gpu-id=0

Code for dumping values :

void parse_ychannel_row_major(uint32_t width, uint32_t height,uint8_t *Y,uint32_t Y_stride)
{
	printf("\nPrinting parse_ychannel_row_major\n\n");
	int x,y;
	for(y=0; y< height; ++y)
	{
		uint8_t *y_row_ptr=Y+y*Y_stride;

		for(x=0;x<width;++x)
		{
			uint8_t *data_ptr=y_row_ptr+x;
			if(data_ptr != NULL)
			{
				uint8_t data = *(data_ptr);
				printf("Y Pixel Value : %d  x : %d , y : %d \n",data,x,y);
			}
			else
			{
				printf("\n\nparse_ychannel_row_major : Null pointer detected\n\n");
			}
		}
	}
}

...
  printf("Processing Height : %d \n",  dsexample->processing_height);
  printf("Processing Width : %d \n",  dsexample->processing_width);
  printf("Pitch : %d \n",  dsexample->inter_buf->surfaceList[0].pitch);
  printf("isContiguous : %d\n",dsexample->inter_buf->isContiguous);

  parse_ychannel_row_major(dsexample->inter_buf->surfaceList[0].pitch,
			 dsexample->processing_height,
			(uint8_t *) dsexample->inter_buf->surfaceList[0].mappedAddr.addr[0],
			dsexample->inter_buf->surfaceList[0].pitch);

  // Use openCV to remove padding and convert RGBA to BGR. Can be skipped if
  // algorithm can handle padded RGBA data.

  in_mat =
      cv::Mat (dsexample->processing_height, dsexample->processing_width,
      CV_8UC4, dsexample->inter_buf->surfaceList[0].mappedAddr.addr[0],
      dsexample->inter_buf->surfaceList[0].pitch);

  cv::cvtColor (in_mat, *dsexample->cvmat, CV_RGBA2BGR);
  cv::imwrite("test.jpg",*dsexample->cvmat);
  cv::imwrite("test_in_mat.jpg",in_mat);
...

Thanks
debug_proc_1280x720.zip (23.8 MB)

[DaneLLL]

Hi,
Please also configure

full-frame=1

[neophyte1]

Hi DaneLLL,

I am able to get the dumps now sometimes but not all the time. What could be the reason behind this variability? Also, when I try to replicate the logic in “tracking_done_prob” function defined in “deepstream-app.c”, it does not seem to work and crashes. Below is the code:

/**
 * Buffer probe function after tracker.
 */
static GstPadProbeReturn
tracking_done_buf_prob (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
  NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
  guint index = bin->index;
  AppCtx *appCtx = bin->appCtx;
  GstBuffer *buf = (GstBuffer *) info->data;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  if (!batch_meta) {
    NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
    return GST_PAD_PROBE_OK;
  }

  GstMapInfo in_map_info;
  NvBufSurface *surface = NULL;

  memset (&in_map_info, 0, sizeof (in_map_info));
  if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
    g_print ("Error: Failed to map gst buffer\n");
   gst_buffer_unmap (buf, &in_map_info);
   return -1;
  }

  cudaError_t cuda_err;

  NvBufSurfTransformRect src_rect, dst_rect;
  surface = (NvBufSurface *) in_map_info.data;  
  
  int batch_size= surface->batchSize;
  printf("\nBatch Size : %d\n",batch_size );

  src_rect.top   = 0;
  src_rect.left  = 0;
  src_rect.width = (guint) surface->surfaceList[0].width;
  src_rect.height= (guint) surface->surfaceList[0].height;

  dst_rect.top   = 0;
  dst_rect.left  = 0;
  dst_rect.width = (guint) surface->surfaceList[0].width;
  dst_rect.height= (guint) surface->surfaceList[0].height;

  NvBufSurfTransformParams nvbufsurface_params;
  nvbufsurface_params.src_rect = &src_rect;
  nvbufsurface_params.dst_rect = &dst_rect;
  nvbufsurface_params.transform_flag =  NVBUFSURF_TRANSFORM_CROP_SRC |	NVBUFSURF_TRANSFORM_CROP_DST;
  nvbufsurface_params.transform_filter = NvBufSurfTransformInter_Default;

NvBufSurface *dst_surface;
  NvBufSurfaceCreateParams nvbufsurface_create_params;

  if (dst_surface)
    NvBufSurfaceDestroy (dst_surface);
  dst_surface = NULL;

  /* An intermediate buffer for NV12/RGBA to BGR conversion  will be
   * required. Can be skipped if custom algorithm can work directly on NV12/RGBA. */
  nvbufsurface_create_params.gpuId  = surface->gpuId;
  nvbufsurface_create_params.width  = (gint) surface->surfaceList[0].width;
  nvbufsurface_create_params.height = (gint) surface->surfaceList[0].height;
  nvbufsurface_create_params.size = 0;
  nvbufsurface_create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
  nvbufsurface_create_params.layout = NVBUF_LAYOUT_PITCH;
  nvbufsurface_create_params.memType = NVBUF_MEM_DEFAULT;

  cuda_err = cudaSetDevice (surface->gpuId);
  printf("\cudaSetDevice : %d\n",cuda_err);

  cudaStream_t cuda_stream;

  cuda_err=cudaStreamCreate (&cuda_stream);

  printf("\cudaStreamCreate : %d\n",cuda_err);

  int create_result = NvBufSurfaceCreate(&dst_surface,batch_size,&nvbufsurface_create_params);	
  
  printf("\ncreate_result : %d\n", create_result);

  NvBufSurfTransformConfigParams transform_config_params;
  NvBufSurfTransform_Error err;

  transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
  transform_config_params.gpu_id = surface->gpuId;
  transform_config_params.cuda_stream = cuda_stream;

  err = NvBufSurfTransformSetSessionParams (&transform_config_params);
  printf("\nError code returned by set session params : %d\n",err);

  NvBufSurfaceMemSet (dst_surface, 0, 0, 0);

  err = NvBufSurfTransform (surface, dst_surface, &nvbufsurface_params);
  if (err != NvBufSurfTransformError_Success) {
	  g_print ("NvBufSurfTransform failed with error %d while converting buffer\n", err);
  }

  printf("\nNvBufSurfTransform_Error : %d\n",err );

  NvBufSurfaceMap (dst_surface, 0, 0, NVBUF_MAP_READ);

  NvBufSurfaceSyncForCpu (dst_surface, 0, 0);

  void *host_rgb_buf;
  int rgb_bytes_per_pixel=3;

  cuda_err = cudaMallocHost (&host_rgb_buf,nvbufsurface_create_params.width * nvbufsurface_create_params.height * rgb_bytes_per_pixel );

  cv::Mat bgr_frame = cv::Mat ( nvbufsurface_create_params.height, nvbufsurface_create_params.width,
      CV_8UC3, host_rgb_buf,
      nvbufsurface_create_params.width * rgb_bytes_per_pixel);

  cv::Mat in_mat =
      cv::Mat (nvbufsurface_create_params.height, nvbufsurface_create_params.width,
      CV_8UC4, dst_surface->surfaceList[0].mappedAddr.addr[0],
      dst_surface->surfaceList[0].pitch);

  cv::cvtColor (in_mat, bgr_frame, CV_RGBA2BGR);
  char string_to_print[200];
  sprintf(string_to_print,"test_%lu.jpg", (int)time(NULL));

  cv::imwrite(string_to_print,bgr_frame);
  cv::imwrite("test_in_mat.jpg",in_mat);

  NvBufSurfaceUnMap (dst_surface, 0, 0);

  /*
   * Output KITTI labels with tracking ID if configured to do so.
   */
  write_kitti_track_output(appCtx, batch_meta);

if (appCtx->primary_bbox_generated_cb)
    appCtx->primary_bbox_generated_cb (appCtx, buf, batch_meta, index);
  return GST_PAD_PROBE_OK;
}

All the error prints return 0 (success)

The program crashes and gives segmentation fault at the imwrite calls. Kindly let me know if you can sight any errors in this code.

Thanks.

[DaneLLL]

Hi,
for more information, does it crash in first or second imwrite()?

[DaneLLL]

Hi,
Below code looks not right.

NvBufSurface *dst_surface;
NvBufSurfaceCreateParams nvbufsurface_create_params;

if (dst_surface)
NvBufSurfaceDestroy (dst_surface);
dst_surface = NULL;

It would be better to destroy the surface and cuda memory at the end of tracking_done_prob

[neophyte1]

Hi DaneLLL,

The crash is observed in the first imwrite call. I will try your suggestion and report.

Thanks

[neophyte1]

Hi DaneLLL,

Still does not work.

[DaneLLL]

Hi,
Please share changes in deepstream_sdk_v4.0_jetson/sources/apps/sample_apps/deepstream-app/Makefile
So that we can build and run the app to reproduce the failure.

[neophyte1]

Hi DaneLLL,

Please find the changes below:

APP:= deepstream-app

#TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -)
TARGET_DEVICE = $(shell g++ -dumpmachine | cut -f1 -d -)

NVDS_VERSION:=4.0

LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(NVDS_VERSION)/lib/

ifeq ($(TARGET_DEVICE),aarch64)
  CFLAGS:= -DPLATFORM_TEGRA
endif

CUDA_VER:=10.0

SRCS:= $(wildcard *.c)
SRCS+= $(wildcard ../../apps-common/src/*.c)

INCS:= $(wildcard *.h)

PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11 opencv

OBJS:= $(SRCS:.c=.o)
CC := g++
  
CFLAGS+= -I../../apps-common/includes -I../../../includes -I/home/edgetensor/Documents/external/curl/include -I/usr/src/nvidia/tegra_multimedia_api/include/ -I/usr/local/cuda-10.0/targets/aarch64-linux/include/  /usr/local/cuda-$(CUDA_VER)/include -I/home/edgetensor/Documents/external/libjpeg/ -I/home/edgetensor/Documents/external/libjpeg/build  -DDS_VERSION_MINOR=0 -DDS_VERSION_MAJOR=4 -fpermissive -Wnarrowing 

LIBS+= -L$(LIB_INSTALL_DIR) -L/home/edgetensor/Documents/external/curl/build/lib -L/usr/local/cuda-10.0/targets/aarch64-linux/lib -L/home/edgetensor/Documents/external/libjpeg/build  -L/usr/local/cuda-$(CUDA_VER)/lib64/  -lcudart -ljpeg  -lnppicc -lnvdsgst_meta -lnvbufsurface -lnvbufsurftransform -lnvds_meta -lnvdsgst_helper -lnvds_utils -lm -lcurl \
       -lgstrtspserver-1.0 -Wl,-rpath,$(LIB_INSTALL_DIR)

CFLAGS+= `pkg-config --cflags $(PKGS)`

LIBS+= `pkg-config --libs $(PKGS)`

all: $(APP)

%.o: %.c $(INCS) Makefile
	$(CC) -c -o $@ $(CFLAGS) $<

$(APP): $(OBJS) Makefile
	$(CC) -o $(APP) $(OBJS) $(LIBS)

clean:
	rm -rf $(OBJS) $(APP)

Thanks.

[neophyte1]

Hi DaneLLL,

I managed to get rid of the segmentation fault by replacing the imwrite calls by cvSaveImage. Kindly find the code below:

...
  cv::cvtColor (in_mat, *bgr_frame, CV_RGBA2BGR);
  CvMat cvMat = *bgr_frame;
  cvSaveImage( "saveimage_dump.jpg", &cvMat );
...

I am attaching a sample dump so that you can relate to the issue better. Kindly point me out how to rectify this.

Thanks.

[DaneLLL]

Hi,
Adding the following code to tracking_done_buf_prob() works fine. For your reference.

#if 1
  static int dump = 0;
  if (dump < 150) {
    GstMapInfo in_map_info;
    NvBufSurface *surface = NULL;

    memset (&in_map_info, 0, sizeof (in_map_info));
    if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
      g_print ("Error: Failed to map gst buffer\n");
      gst_buffer_unmap (buf, &in_map_info);
      return GST_PAD_PROBE_OK;
    }
    cudaError_t cuda_err;

    NvBufSurfTransformRect src_rect, dst_rect;
    surface = (NvBufSurface *) in_map_info.data;  
  
    int batch_size= surface->batchSize;
    printf("\nBatch Size : %d, resolution : %dx%d \n",batch_size,
        surface->surfaceList[0].width, surface->surfaceList[0].height);

    src_rect.top   = 0;
    src_rect.left  = 0;
    src_rect.width = (guint) surface->surfaceList[0].width;
    src_rect.height= (guint) surface->surfaceList[0].height;

    dst_rect.top   = 0;
    dst_rect.left  = 0;
    dst_rect.width = (guint) surface->surfaceList[0].width;
    dst_rect.height= (guint) surface->surfaceList[0].height;

    NvBufSurfTransformParams nvbufsurface_params;
    nvbufsurface_params.src_rect = &src_rect;
    nvbufsurface_params.dst_rect = &dst_rect;
    nvbufsurface_params.transform_flag =  NVBUFSURF_TRANSFORM_CROP_SRC | NVBUFSURF_TRANSFORM_CROP_DST;
    nvbufsurface_params.transform_filter = NvBufSurfTransformInter_Default;
  
    NvBufSurface *dst_surface = NULL;
    NvBufSurfaceCreateParams nvbufsurface_create_params;

    /* An intermediate buffer for NV12/RGBA to BGR conversion  will be
     * required. Can be skipped if custom algorithm can work directly on NV12/RGBA. */
    nvbufsurface_create_params.gpuId  = surface->gpuId;
    nvbufsurface_create_params.width  = (gint) surface->surfaceList[0].width;
    nvbufsurface_create_params.height = (gint) surface->surfaceList[0].height;
    nvbufsurface_create_params.size = 0;
    nvbufsurface_create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
    nvbufsurface_create_params.layout = NVBUF_LAYOUT_PITCH;
    nvbufsurface_create_params.memType = NVBUF_MEM_DEFAULT;

    cuda_err = cudaSetDevice (surface->gpuId);

    cudaStream_t cuda_stream;

    cuda_err=cudaStreamCreate (&cuda_stream);

    int create_result = NvBufSurfaceCreate(&dst_surface,batch_size,&nvbufsurface_create_params);	

    NvBufSurfTransformConfigParams transform_config_params;
    NvBufSurfTransform_Error err;

    transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
    transform_config_params.gpu_id = surface->gpuId;
    transform_config_params.cuda_stream = cuda_stream;
    err = NvBufSurfTransformSetSessionParams (&transform_config_params);

    NvBufSurfaceMemSet (dst_surface, 0, 0, 0);
    err = NvBufSurfTransform (surface, dst_surface, &nvbufsurface_params);
    if (err != NvBufSurfTransformError_Success) {
  	  g_print ("NvBufSurfTransform failed with error %d while converting buffer\n", err);
    }
    NvBufSurfaceMap (dst_surface, 0, 0, NVBUF_MAP_READ);
    NvBufSurfaceSyncForCpu (dst_surface, 0, 0);

    cv::Mat bgr_frame = cv::Mat (cv::Size(nvbufsurface_create_params.width, nvbufsurface_create_params.height), CV_8UC3);

    cv::Mat in_mat =
        cv::Mat (nvbufsurface_create_params.height, nvbufsurface_create_params.width,
        CV_8UC4, dst_surface->surfaceList[0].mappedAddr.addr[0],
        dst_surface->surfaceList[0].pitch);

    cv::cvtColor (in_mat, bgr_frame, CV_RGBA2BGR);

    char filename[64];
    snprintf(filename, 64, "/tmp/image%03d.jpg", dump);
    cv::imwrite(filename,bgr_frame);
    dump ++;

    NvBufSurfaceUnMap (dst_surface, 0, 0);
    NvBufSurfaceDestroy (dst_surface);
    cudaStreamDestroy (cuda_stream);
    gst_buffer_unmap (buf, &in_map_info);
  }
#endif

[neophyte1]

Thanks DaneLLL. This seems to work. However, could you please explain conceptually what went wrong earlier?