Getting always gray frame in place of rgb frame from deepstream-test3.c

i want to save rgb frame from deepstream-test3.c but i get always a gray image , i changed the input video and save function many time but i get also the same result , i think it’s due to error in getting image data , i use jetson xavier jetpack 4.3 , when i used libjpeg to save data and use rgb config i get in the same image 3 times gray scale image (like that file_y_1920x1080_0|690x388) i used the code below , please help me

  		/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.
	 *
	 * Permission is hereby granted, free of charge, to any person obtaining a
	 * copy of this software and associated documentation files (the "Software"),
	 * to deal in the Software without restriction, including without limitation
	 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
	 * and/or sell copies of the Software, and to permit persons to whom the
	 * Software is furnished to do so, subject to the following conditions:
	 *
	 * The above copyright notice and this permission notice shall be included in
	 * all copies or substantial portions of the Software.
	 *
	 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
	 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
	 * DEALINGS IN THE SOFTWARE.
	 */

	#include <gst/gst.h>
	#include <glib.h>
	#include <stdio.h>
	#include <math.h>
	#include <string.h>
	#include <sys/time.h>
	#include <jpeglib.h>


	#include "nvbufsurface.h"
	#include "/usr/local/cuda-10.0/targets/aarch64-linux/include/cuda_runtime.h"
	#include "/usr/local/cuda-10.0/targets/aarch64-linux/include/device_launch_parameters.h"

	#include "gstnvdsmeta.h"
	//#include "gstnvstreammeta.h"
	#ifndef PLATFORM_TEGRA
	#include "gst-nvmessage.h"
	#endif

	#define MAX_DISPLAY_LEN 64

	#define PGIE_CLASS_ID_VEHICLE 0
	#define PGIE_CLASS_ID_PERSON 2

	/* The muxer output resolution must be set if the input streams will be of
	 * different resolution. The muxer will scale all the input frames to this
	 * resolution. */
	#define MUXER_OUTPUT_WIDTH 1920
	#define MUXER_OUTPUT_HEIGHT 1080

	/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set
	 * based on the fastest source's framerate. */
	#define MUXER_BATCH_TIMEOUT_USEC 4000000

	#define TILED_OUTPUT_WIDTH 1920
	#define TILED_OUTPUT_HEIGHT 1080

	/* NVIDIA Decoder source pad memory feature. This feature signifies that source
	 * pads having this capability will push GstBuffers containing cuda buffers. */
	#define GST_CAPS_FEATURES_NVMM "memory:NVMM"
	int txtnumb=0;
	gchar pgie_classes_str[4][32] = { "Vehicle", "TwoWheeler", "Person",
	  "RoadSign"
	};

	#define FPS_PRINT_INTERVAL 300
	//static struct timeval start_time = { };

	//static guint probe_counter = 0;

	/* tiler_sink_pad_buffer_probe  will extract metadata received on OSD sink pad
	 * and update params for drawing rectangle, object information etc. */

	int outnumber=0;

	int write_jpeg_file( char *filename, unsigned char* rgb_image , int width, int height, int bytes_per_pixel, J_COLOR_SPACE color_space )
	{
		struct jpeg_compress_struct cinfo;
		struct jpeg_error_mgr jerr;
		JSAMPROW row_pointer[1];
		FILE *outfile = fopen( filename, "wb" );
		
		if ( !outfile )
		{
			printf("Error opening output jpeg file %s\n!", filename );
			return -1;
		}
		cinfo.err = jpeg_std_error( &jerr );
		jpeg_create_compress(&cinfo);
		jpeg_stdio_dest(&cinfo, outfile);


		cinfo.image_width = width;	
		cinfo.image_height = height;
		cinfo.input_components =bytes_per_pixel;
		cinfo.in_color_space =   color_space; //JCS_RGB

		jpeg_set_defaults( &cinfo );
		
		jpeg_start_compress( &cinfo, TRUE );
	    
		while( cinfo.next_scanline < cinfo.image_height )
		{
			row_pointer[0] = &rgb_image[ cinfo.next_scanline * cinfo.image_width *  cinfo.input_components];
			jpeg_write_scanlines( &cinfo, row_pointer, 1 );
		}

		jpeg_finish_compress( &cinfo );
		jpeg_destroy_compress( &cinfo );
		fclose( outfile );

		return 1;
	}


	static GstPadProbeReturn
	tiler_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info,
	    gpointer u_data)
	{
	    GstBuffer *buf = (GstBuffer *) info->data;
	    guint num_rects = 0; 
	    NvDsObjectMeta *obj_meta = NULL;
	    guint vehicle_count = 0;
	    guint person_count = 0;
	    NvDsMetaList * l_frame = NULL;
	    NvDsMetaList * l_obj = NULL;
	  	NvDsUserMeta *user_meta = NULL;
	    FILE *fp1;
	    //NvDsDisplayMeta *display_meta = NULL;
	    char *txtfilename=NULL;
	    NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
	   
	    GstMapInfo in_map_info;
	    NvBufSurface *surface = NULL;
	    int imgpix=0;
	    int txtstep;
	    unsigned char *totalpix;
	    char *totalpix1;
	    unsigned int *totalpix2;
	    long unsigned int *totalpix3;
	    char* src_data = NULL;
		  memset (&in_map_info, 0, sizeof (in_map_info));
		  if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
		    g_printerr ("Error: Failed to map gst buffer\n");
		  }

		  

	      for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
		l_frame = l_frame->next) {
		NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
		surface = (NvBufSurface *) in_map_info.data;
		uint32_t frame_width  =  surface->surfaceList[frame_meta->batch_id].width;
		uint32_t frame_height =  surface->surfaceList[frame_meta->batch_id].height;
		uint32_t Y_stride     =  surface->surfaceList[frame_meta->batch_id].pitch;
		uint32_t buffer_size = surface->surfaceList[frame_meta->batch_id].dataSize;
		uint32_t est_size = frame_width*frame_height;

		src_data = (char*) malloc(surface->surfaceList[frame_meta->batch_id].dataSize);
		if(src_data == NULL) {
		    g_print("Error: failed to malloc src_data \n");
		    continue;
		}

		NvBufSurfaceMap (surface, -1, -1, NVBUF_MAP_READ);
		NvBufSurfacePlaneParams *pParams = &surface->surfaceList[frame_meta->batch_id].planeParams;
		unsigned int offset = 0;
		for(unsigned int num_planes=0; num_planes < pParams->num_planes; num_planes++){
		    if(num_planes>0)
			offset += pParams->height[num_planes-1]*(pParams->bytesPerPix[num_planes-1]*pParams->width[num_planes-1]);
		    for (unsigned int h = 0; h < pParams->height[num_planes]; h++) {
		     memcpy((void *)(src_data+offset+h*pParams->bytesPerPix[num_planes]*pParams->width[num_planes]),
			    (void *)((char *)surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[num_planes]+h*pParams->pitch[num_planes]),
			    pParams->bytesPerPix[num_planes]*pParams->width[num_planes]
			    );
		    }
		}
		NvBufSurfaceSyncForDevice (surface, -1, -1);
		NvBufSurfaceUnMap (surface, -1, -1);

	      

		FILE *fp1;
		char filenametxt[200];
		sprintf(filenametxt,"file_y_%dx%d_%d.txt",1920,1080,outnumber);
	      
		char filename[200];
		sprintf(filename,"file_y_%dx%d_%d.jpg",frame_width,frame_height,outnumber);
		outnumber=outnumber+1;
		printf("filename:%s\n",filename);
	       
		write_jpeg_file( filename, (unsigned char*) src_data , frame_width, frame_height, 3, JCS_RGB );
	    
		

		if(src_data != NULL) {
		    free(src_data);
		    src_data = NULL;
		  }
		


		//int offset = 0;
		for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;
			l_obj = l_obj->next) {
		    obj_meta = (NvDsObjectMeta *) (l_obj->data);
		    if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) {
			vehicle_count++;
			num_rects++;
		    }
		    if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) {
			person_count++;
			num_rects++;
		    }
		}
	       
	#if 0
		display_meta = nvds_acquire_display_meta_from_pool(batch_meta);
		NvOSD_TextParams *txt_params  = &display_meta->text_params;
		txt_params->display_text = g_malloc0 (MAX_DISPLAY_LEN);
		offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Person = %d ", person_count);
		offset = snprintf(txt_params->display_text + offset , MAX_DISPLAY_LEN, "Vehicle = %d ", vehicle_count);

		/* Now set the offsets where the string should appear */
		txt_params->x_offset = 10;
		txt_params->y_offset = 12;

		/* Font , font-color and font-size */
		txt_params->font_params.font_name = "Serif";
		txt_params->font_params.font_size = 10;
		txt_params->font_params.font_color.red = 1.0;
		txt_params->font_params.font_color.green = 1.0;
		txt_params->font_params.font_color.blue = 1.0;
		txt_params->font_params.font_color.alpha = 1.0;

		/* Text background color */
		txt_params->set_bg_clr = 1;
		txt_params->text_bg_clr.red = 0.0;
		txt_params->text_bg_clr.green = 0.0;
		txt_params->text_bg_clr.blue = 0.0;
		txt_params->text_bg_clr.alpha = 1.0;

		nvds_add_display_meta_to_frame(frame_meta, display_meta);
	#endif

	    }
	    gst_buffer_unmap (buf, &in_map_info);

	    return GST_PAD_PROBE_OK;
	}

	static gboolean
	bus_call (GstBus * bus, GstMessage * msg, gpointer data)
	{
	  GMainLoop *loop = (GMainLoop *) data;
	  switch (GST_MESSAGE_TYPE (msg)) {
	    case GST_MESSAGE_EOS:
	      g_print ("End of stream\n");
	      g_main_loop_quit (loop);
	      break;
	    case GST_MESSAGE_WARNING:
	    {
	      gchar *debug;
	      GError *error;
	      gst_message_parse_warning (msg, &error, &debug);
	      g_printerr ("WARNING from element %s: %s\n",
		  GST_OBJECT_NAME (msg->src), error->message);
	      g_free (debug);
	      g_printerr ("Warning: %s\n", error->message);
	      g_error_free (error);
	      break;
	    }
	    case GST_MESSAGE_ERROR:
	    {
	      gchar *debug;
	      GError *error;
	      gst_message_parse_error (msg, &error, &debug);
	      g_printerr ("ERROR from element %s: %s\n",
		  GST_OBJECT_NAME (msg->src), error->message);
	      if (debug)
		g_printerr ("Error details: %s\n", debug);
	      g_free (debug);
	      g_error_free (error);
	      g_main_loop_quit (loop);
	      break;
	    }
	#ifndef PLATFORM_TEGRA
	    case GST_MESSAGE_ELEMENT:
	    {
	      if (gst_nvmessage_is_stream_eos (msg)) {
		guint stream_id;
		if (gst_nvmessage_parse_stream_eos (msg, &stream_id)) {
		  g_print ("Got EOS from stream %d\n", stream_id);
		}
	      }
	      break;
	    }
	#endif
	    default:
	      break;
	  }
	  return TRUE;
	}

	static void
	cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data)
	{
	  g_print ("In cb_newpad\n");
	  GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad);
	  const GstStructure *str = gst_caps_get_structure (caps, 0);
	  const gchar *name = gst_structure_get_name (str);
	  GstElement *source_bin = (GstElement *) data;
	  GstCapsFeatures *features = gst_caps_get_features (caps, 0);

	  /* Need to check if the pad created by the decodebin is for video and not
	   * audio. */
	  if (!strncmp (name, "video", 5)) {
	    /* Link the decodebin pad only if decodebin has picked nvidia
	     * decoder plugin nvdec_*. We do this by checking if the pad caps contain
	     * NVMM memory features. */
	    if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) {
	      /* Get the source bin ghost pad */
	      GstPad *bin_ghost_pad = gst_element_get_static_pad (source_bin, "src");
	      if (!gst_ghost_pad_set_target (GST_GHOST_PAD (bin_ghost_pad),
		      decoder_src_pad)) {
		g_printerr ("Failed to link decoder src pad to source bin ghost pad\n");
	      }
	      gst_object_unref (bin_ghost_pad);
	    } else {
	      g_printerr ("Error: Decodebin did not pick nvidia decoder plugin.\n");
	    }
	  }
	}

	static void
	decodebin_child_added (GstChildProxy * child_proxy, GObject * object,
	    gchar * name, gpointer user_data)
	{
	  g_print ("Decodebin child added: %s\n", name);
	  if (g_strrstr (name, "decodebin") == name) {
	    g_signal_connect (G_OBJECT (object), "child-added",
		G_CALLBACK (decodebin_child_added), user_data);
	  }
	  if (g_strstr_len (name, -1, "nvv4l2decoder") == name) {
	    g_print ("Seting bufapi_version\n");
	    g_object_set (object, "bufapi-version", TRUE, NULL);
	  }
	}

	static GstElement *
	create_source_bin (guint index, gchar * uri)
	{
	  GstElement *bin = NULL, *uri_decode_bin = NULL;
	  gchar bin_name[16] = { };

	  g_snprintf (bin_name, 15, "source-bin-%02d", index);
	  /* Create a source GstBin to abstract this bin's content from the rest of the
	   * pipeline */
	  bin = gst_bin_new (bin_name);

	  /* Source element for reading from the uri.
	   * We will use decodebin and let it figure out the container format of the
	   * stream and the codec and plug the appropriate demux and decode plugins. */
	  uri_decode_bin = gst_element_factory_make ("uridecodebin", "uri-decode-bin");

	  if (!bin || !uri_decode_bin) {
	    g_printerr ("One element in source bin could not be created.\n");
	    return NULL;
	  }

	  /* We set the input uri to the source element */
	  g_object_set (G_OBJECT (uri_decode_bin), "uri", uri, NULL);

	  /* Connect to the "pad-added" signal of the decodebin which generates a
	   * callback once a new pad for raw data has beed created by the decodebin */
	  g_signal_connect (G_OBJECT (uri_decode_bin), "pad-added",
	      G_CALLBACK (cb_newpad), bin);
	  g_signal_connect (G_OBJECT (uri_decode_bin), "child-added",
	      G_CALLBACK (decodebin_child_added), bin);

	  gst_bin_add (GST_BIN (bin), uri_decode_bin);

	  /* We need to create a ghost pad for the source bin which will act as a proxy
	   * for the video decoder src pad. The ghost pad will not have a target right
	   * now. Once the decode bin creates the video decoder and generates the
	   * cb_newpad callback, we will set the ghost pad target to the video decoder
	   * src pad. */
	  if (!gst_element_add_pad (bin, gst_ghost_pad_new_no_target ("src",
		      GST_PAD_SRC))) {
	    g_printerr ("Failed to add ghost pad in source bin\n");
	    return NULL;
	  }

	  return bin;
	}

	int
	main (int argc, char *argv[])
	{
	  GMainLoop *loop = NULL;
	  GstElement *pipeline = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL,
	      *nvvidconv = NULL, *nvosd = NULL, *tiler = NULL;
	#ifdef PLATFORM_TEGRA
	  GstElement *transform = NULL;
	#endif
	  GstBus *bus = NULL;
	  guint bus_watch_id;
	  GstPad *tiler_src_pad = NULL;
	  guint i, num_sources;
	  guint tiler_rows, tiler_columns;
	  guint pgie_batch_size;

	  /* Check input arguments */
	  if (argc < 2) {
	    g_printerr ("Usage: %s <uri1> [uri2] ... [uriN] \n", argv[0]);
	    return -1;
	  }
	  num_sources = argc - 1;

	  /* Standard GStreamer initialization */
	  gst_init (&argc, &argv);
	  loop = g_main_loop_new (NULL, FALSE);

	  /* Create gstreamer elements */
	  /* Create Pipeline element that will form a connection of other elements */
	  pipeline = gst_pipeline_new ("dstest3-pipeline");

	  /* Create nvstreammux instance to form batches from one or more sources. */
	  streammux = gst_element_factory_make ("nvstreammux", "stream-muxer");

	  if (!pipeline || !streammux) {
	    g_printerr ("One element could not be created. Exiting.\n");
	    return -1;
	  }
	  gst_bin_add (GST_BIN (pipeline), streammux);

	  for (i = 0; i < num_sources; i++) {
	    GstPad *sinkpad, *srcpad;
	    gchar pad_name[16] = { };
	    GstElement *source_bin = create_source_bin (i, argv[i + 1]);

	    if (!source_bin) {
	      g_printerr ("Failed to create source bin. Exiting.\n");
	      return -1;
	    }

	    gst_bin_add (GST_BIN (pipeline), source_bin);

	    g_snprintf (pad_name, 15, "sink_%u", i);
	    sinkpad = gst_element_get_request_pad (streammux, pad_name);
	    if (!sinkpad) {
	      g_printerr ("Streammux request sink pad failed. Exiting.\n");
	      return -1;
	    }

	    srcpad = gst_element_get_static_pad (source_bin, "src");
	    if (!srcpad) {
	      g_printerr ("Failed to get src pad of source bin. Exiting.\n");
	      return -1;
	    }

	    if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) {
	      g_printerr ("Failed to link source bin to stream muxer. Exiting.\n");
	      return -1;
	    }

	    gst_object_unref (srcpad);
	    gst_object_unref (sinkpad);
	  }

	  /* Use nvinfer to infer on batched frame. */
	  pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine");

	  /* Use nvtiler to composite the batched frames into a 2D tiled array based
	   * on the source of the frames. */
	  tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler");

	  /* Use convertor to convert from NV12 to RGBA as required by nvosd */
	  nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter");

	  /* Create OSD to draw on the converted RGBA buffer */
	  nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay");

	  /* Finally render the osd output */
	#ifdef PLATFORM_TEGRA
	  transform = gst_element_factory_make ("nvegltransform", "nvegl-transform");
	#endif
	  sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer");

	  if (!pgie || !tiler || !nvvidconv || !nvosd || !sink) {
	    g_printerr ("One element could not be created. Exiting.\n");
	    return -1;
	  }

	#ifdef PLATFORM_TEGRA
	  if(!transform) {
	    g_printerr ("One tegra element could not be created. Exiting.\n");
	    return -1;
	  }
	#endif

	  g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height",
	      MUXER_OUTPUT_HEIGHT, "batch-size", num_sources,
	      "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL);

	  /* Configure the nvinfer element using the nvinfer config file. */
	  g_object_set (G_OBJECT (pgie),
	      "config-file-path", "dstest3_pgie_config.txt", NULL);

	  /* Override the batch-size set in the config file with the number of sources. */
	  g_object_get (G_OBJECT (pgie), "batch-size", &pgie_batch_size, NULL);
	  if (pgie_batch_size != num_sources) {
	    g_printerr
		("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n",
		pgie_batch_size, num_sources);
	    g_object_set (G_OBJECT (pgie), "batch-size", num_sources, NULL);
	  }

	  tiler_rows = (guint) sqrt (num_sources);
	  tiler_columns = (guint) ceil (1.0 * num_sources / tiler_rows);
	  /* we set the tiler properties here */
	  g_object_set (G_OBJECT (tiler), "rows", tiler_rows, "columns", tiler_columns,
	      "width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL);

	  /* we add a message handler */
	  bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
	  bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
	  gst_object_unref (bus);

	  /* Set up the pipeline */
	  /* we add all elements into the pipeline */
	#ifdef PLATFORM_TEGRA
	  gst_bin_add_many (GST_BIN (pipeline), pgie, tiler, nvvidconv, nvosd, transform, sink,
	      NULL);
	  /* we link the elements together
	   * nvstreammux -> nvinfer -> nvtiler -> nvvidconv -> nvosd -> video-renderer */
	  if (!gst_element_link_many (streammux, pgie, tiler, nvvidconv, nvosd, transform, sink,
		  NULL)) {
	    g_printerr ("Elements could not be linked. Exiting.\n");
	    return -1;
	  }
	#else
	gst_bin_add_many (GST_BIN (pipeline), pgie, tiler, nvvidconv, nvosd, sink,
	      NULL);
	  /* we link the elements together
	   * nvstreammux -> nvinfer -> nvtiler -> nvvidconv -> nvosd -> video-renderer */
	  if (!gst_element_link_many (streammux, pgie, tiler, nvvidconv, nvosd, sink,
		  NULL)) {
	    g_printerr ("Elements could not be linked. Exiting.\n");
	    return -1;
	  }
	#endif

	  /* Lets add probe to get informed of the meta data generated, we add probe to
	   * the sink pad of the osd element, since by that time, the buffer would have
	   * had got all the metadata. */
	  tiler_src_pad = gst_element_get_static_pad (pgie, "src");
	  if (!tiler_src_pad)
	    g_print ("Unable to get src pad\n");
	  else
	    gst_pad_add_probe (tiler_src_pad, GST_PAD_PROBE_TYPE_BUFFER,
		tiler_src_pad_buffer_probe, NULL, NULL);

	  /* Set the pipeline to "playing" state */
	  g_print ("Now playing:");
	  for (i = 0; i < num_sources; i++) {
	    g_print (" %s,", argv[i + 1]);
	  }
	  g_print ("\n");
	  gst_element_set_state (pipeline, GST_STATE_PLAYING);

	  /* Wait till pipeline encounters an error or EOS */
	  g_print ("Running...\n");
	  g_main_loop_run (loop);

	  /* Out of the main loop, clean up nicely */
	  g_print ("Returned, stopping playback\n");
	  gst_element_set_state (pipeline, GST_STATE_NULL);
	  g_print ("Deleting pipeline\n");
	  gst_object_unref (GST_OBJECT (pipeline));
	  g_source_remove (bus_watch_id);
	  g_main_loop_unref (loop);
	  return 0;
	}!

Please try with the latest DeepStream SDK 5.0.

https://forums.developer.nvidia.com/t/announcing-developer-preview-for-deepstream-5-0/121619

1 Like

Hi,
Please check if you can see video playback in running:

$ gst-launch-1.0 uridecodebin uri=_URI_ADDR_ ! nvoverlaysink

Would like ot know if video decoder can work with source.