Hi,
I am running the deepstream segmenation application in DS 4.0. I would like to write the output to a file and visualize it. Are there any examples or code snippets to view the output segmentation results for this example?
-SN
Hi,
I am running the deepstream segmenation application in DS 4.0. I would like to write the output to a file and visualize it. Are there any examples or code snippets to view the output segmentation results for this example?
-SN
What’s platform you used, you should observe the result from the output screen when you running the app.
I am running the application in a docker on a remote server (Tesla V100). It is ideal to write the output to a file. I have set the properties as: g_object_set (G_OBJECT (sink), “location”, “segmentation-out.mjpeg”, NULL); Any thoughts?
If you set the output sink to a file, then you can play the file to see the result.
BTW, you can add dump in tiler_src_pad_buffer_probe, paste the code for your reference.
#ifdef DUMP_JPG
static unsigned char class2BGR[] = {
0, 0, 0, 0, 0, 128, 0, 128, 0,
0, 128, 128, 128, 0, 0, 128, 0, 128,
128, 128, 0, 128, 128, 128, 0, 0, 64,
0, 0, 192, 0, 128, 64, 0, 128, 192,
128, 0, 64, 128, 0, 192, 128, 128, 64,
128, 128, 192, 0, 64, 0, 0, 64, 128,
0, 192, 0, 0, 192, 128, 128, 64, 0,
192, 192, 0
};
static cv::Mat overlayColor(int* mask, int height, int width) {
unsigned char* buffer_R = reinterpret_cast<unsigned char*>(malloc(sizeof(unsigned char) * height * width));
unsigned char* buffer_G = reinterpret_cast<unsigned char*>(malloc(sizeof(unsigned char) * height * width));
unsigned char* buffer_B = reinterpret_cast<unsigned char*>(malloc(sizeof(unsigned char) * height * width));
unsigned char* buffer_A = reinterpret_cast<unsigned char*>(malloc(sizeof(unsigned char) * height * width));
g_print("buffer size = %d\n", (height * width));
for(int pix_id = 0; pix_id < width * height; pix_id++) {
g_print("mask[pix_id] = %d\n", mask[pix_id]);
unsigned char* color = class2BGR + ((mask[pix_id] + 3)* 3);
buffer_B[pix_id] = color[0];
buffer_G[pix_id] = color[1];
buffer_R[pix_id] = color[2];
buffer_A[pix_id] = 255;
}
std::vector<cv::Mat> channels;
channels.push_back(cv::Mat(height, width, CV_8UC1, buffer_B));
channels.push_back(cv::Mat(height, width, CV_8UC1, buffer_G));
channels.push_back(cv::Mat(height, width, CV_8UC1, buffer_R));
channels.push_back(cv::Mat(height, width, CV_8UC1, buffer_A));
cv::Mat merged;
cv::merge(channels, merged);
free(buffer_B);
free(buffer_G);
free(buffer_R);
return merged;
}
static void dump_jpg(char* src_data, int src_width, int src_height, NvDsInferSegmentationMeta* seg_meta, int source_id, int frame_number) {
char file_name[128];
cv::Mat map_mat = overlayColor((int*)seg_meta->class_map, seg_meta->height, seg_meta->width);
sprintf(file_name, "dump_map_stream%2d_frame%03d.jpg", source_id, frame_number);
cv::Mat map_resized;
cv::resize(map_mat, map_resized, cv::Size(src_width, src_height));
cv::imwrite(file_name, map_resized);
// NV12 source data
cv::Mat src_mat(src_height *3/2, src_width, CV_8UC1, (void*)src_data);
cv::Mat src_mat_BGRA;
cv::cvtColor(src_mat, src_mat_BGRA, CV_YUV2BGRA_NV12);
sprintf(file_name, "dump_orig_stream%2d_frame%03d.jpg", source_id, frame_number);
cv::imwrite(file_name, src_mat_BGRA);
cv::Mat dst_mat;
float alpha = 0.5;
float beta = 1.0 - alpha;
cv::addWeighted(src_mat_BGRA , alpha, map_resized, beta, 0.0, dst_mat);
sprintf(file_name, "dump_merged_stream%2d_%03d.jpg", source_id, frame_number);
cv::imwrite(file_name, dst_mat);
}
#endif
/* tiler_sink_pad_buffer_probe will extract metadata received on segmentation
* src pad */
static GstPadProbeReturn
tiler_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info,
gpointer u_data)
{
#ifdef DUMP_JPG
GstBuffer *buf = (GstBuffer *) info->data;
NvDsMetaList * l_frame = NULL;
NvDsMetaList * l_user_meta = NULL;
NvDsUserMeta *user_meta = NULL;
NvDsInferSegmentationMeta* seg_meta_data = NULL;
// Get original raw data
GstMapInfo in_map_info;
char* src_data = NULL;
if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
g_print ("Error: Failed to map gst buffer\n");
gst_buffer_unmap (buf, &in_map_info);
return GST_PAD_PROBE_OK;
}
NvBufSurface *surface = (NvBufSurface *)in_map_info.data;
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
l_frame = l_frame->next) {
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
/* Validate user meta */
for (l_user_meta = frame_meta->frame_user_meta_list; l_user_meta != NULL;
l_user_meta = l_user_meta->next) {
user_meta = (NvDsUserMeta *) (l_user_meta->data);
if (user_meta && user_meta->base_meta.meta_type == NVDSINFER_SEGMENTATION_META) {
seg_meta_data = (NvDsInferSegmentationMeta*)user_meta->user_meta_data;
}
}
src_data = (char*) malloc(surface->surfaceList[frame_meta->batch_id].dataSize);
if(src_data == NULL) {
g_print("Error: failed to malloc src_data \n");
continue;
}
cudaMemcpy((void*)src_data,
(void*)surface->surfaceList[frame_meta->batch_id].dataPtr,
surface->surfaceList[frame_meta->batch_id].dataSize,
cudaMemcpyDeviceToHost);
dump_jpg(src_data,
surface->surfaceList[frame_meta->batch_id].width,
surface->surfaceList[frame_meta->batch_id].height,
seg_meta_data, frame_meta->source_id, frame_meta->frame_num);
if(src_data != NULL) {
free(src_data);
src_data = NULL;
}
}
gst_buffer_unmap (buf, &in_map_info);
#endif
return GST_PAD_PROBE_OK;
}
Thank you, I will try this out and post an update.
Hi bcao,
I am trying to use your advice given above. I tried using opencv along with deepstream-app. Please find the Makefile below :
APP:= deepstream-app
#TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -)
TARGET_DEVICE = $(shell g++ -dumpmachine | cut -f1 -d -)
NVDS_VERSION:=4.0
LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(NVDS_VERSION)/lib/
ifeq ($(TARGET_DEVICE),aarch64)
CFLAGS:= -DPLATFORM_TEGRA
endif
CC := gcc
CXX := g++
SRCS:= $(wildcard *.c)
SRCS+= $(wildcard ../../apps-common/src/*.c)
SRCS_CXX:= $(wildcard *.cpp)
INCS:= $(wildcard *.h)
PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11
OBJS:= $(SRCS:.c=.o)
OBJS_CXX:= $(SRCS:.cpp=.co)
CFLAGS+= -I../../apps-common/includes -I../../../includes -I/usr/src/nvidia/tegra_multimedia_api/include/ -I/usr/local/cuda-10.0/targets/aarch64-linux/include/ -DDS_VERSION_MINOR=0 -DDS_VERSION_MAJOR=4
LIBS+= -L$(LIB_INSTALL_DIR) -L/usr/local/cuda-10.0/targets/aarch64-linux/lib -lnppicc -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper -lnvds_utils -lm -lcurl \
-lgstrtspserver-1.0 -Wl,-rpath,$(LIB_INSTALL_DIR)
CFLAGS+= `pkg-config --cflags $(PKGS)`
LIBS+= `pkg-config --libs $(PKGS)`
all: $(APP)
%.o: %.c $(INCS) Makefile
$(CC) -c -o $@ $(CFLAGS) $<
%.co: %.cpp $(INCS) Makefile
$(CXX) -c -o $@ $(CFLAGS) $<
$(APP): $(OBJS_CXX) $(OBJS) Makefile
$(CXX) -o $(APP) $(OBJS_CXX) $(OBJS) $(LIBS)
clean:
rm -rf $(OBJS) $(APP)
However, I get the following error when I build the application:
deepstream_app.c:23:10: fatal error: gst/gst.h: No such file or directory
#include <gst/gst.h>
^~~~~~~~~~~
compilation terminated.
In file included from deepstream_app_main.c:23:0:
deepstream_app.h:31:10: fatal error: gst/gst.h: No such file or directory
#include <gst/gst.h>
^~~~~~~~~~~
compilation terminated.
In file included from deepstream_app_config_parser.c:23:0:
deepstream_app.h:31:10: fatal error: gst/gst.h: No such file or directory
#include <gst/gst.h>
^~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_tracker_bin.c:23:10: fatal error: deepstream_common.h: No such file or directory
#include "deepstream_common.h"
^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_primary_gie_bin.c:26:10: fatal error: deepstream_common.h: No such file or directory
#include "deepstream_common.h"
^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_source_bin.c:25:10: fatal error: gstnvdsmeta.h: No such file or directory
#include "gstnvdsmeta.h"
^~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_config_file_parser.c:23:10: fatal error: deepstream_common.h: No such file or directory
#include "deepstream_common.h"
^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_common.c:23:10: fatal error: gst/gst.h: No such file or directory
#include <gst/gst.h>
^~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_sink_bin.c:30:10: fatal error: deepstream_common.h: No such file or directory
#include "deepstream_common.h"
^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_perf.c:29:10: fatal error: gstnvdsmeta.h: No such file or directory
#include "gstnvdsmeta.h"
^~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_dewarper_bin.c:23:10: fatal error: deepstream_common.h: No such file or directory
#include "deepstream_common.h"
^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_dsexample.c:23:10: fatal error: deepstream_common.h: No such file or directory
#include "deepstream_common.h"
^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_secondary_gie_bin.c:26:10: fatal error: deepstream_common.h: No such file or directory
#include "deepstream_common.h"
^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_tiled_display_bin.c:23:10: fatal error: deepstream_common.h: No such file or directory
#include "deepstream_common.h"
^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_osd_bin.c:23:10: fatal error: deepstream_common.h: No such file or directory
#include "deepstream_common.h"
^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_streammux.c:23:10: fatal error: deepstream_common.h: No such file or directory
#include "deepstream_common.h"
^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
Makefile:69: recipe for target 'deepstream-app' failed
Kindly let me know how to build with opencv. If you can provide the Makefile it will be great help.
Since opencv did not work I used libjpeg to convert only the Y channel of the buffer. However, the output was almost completely black, not representative of my input.
Code:
int write_jpeg_file( char *filename, unsigned char* rgb_image , int width, int height, int bytes_per_pixel, J_COLOR_SPACE color_space )
{
//parse_rgb_row_major(width,height,rgb_image,width*3);
struct jpeg_compress_struct cinfo;
struct jpeg_error_mgr jerr;
JSAMPROW row_pointer[1];
FILE *outfile = fopen( filename, "wb" );
if ( !outfile )
{
printf("Error opening output jpeg file %s\n!", filename );
return -1;
}
cinfo.err = jpeg_std_error( &jerr );
jpeg_create_compress(&cinfo);
jpeg_stdio_dest(&cinfo, outfile);
cinfo.image_width = width;
cinfo.image_height = height;
cinfo.input_components = bytes_per_pixel;
cinfo.in_color_space = color_space; //JCS_RGB
jpeg_set_defaults( &cinfo );
jpeg_start_compress( &cinfo, TRUE );
while( cinfo.next_scanline < cinfo.image_height )
{
row_pointer[0] = &rgb_image[ cinfo.next_scanline * cinfo.image_width * cinfo.input_components];
jpeg_write_scanlines( &cinfo, row_pointer, 1 );
}
jpeg_finish_compress( &cinfo );
jpeg_destroy_compress( &cinfo );
fclose( outfile );
return 1;
}
/**
* Buffer probe function after tracker.
*/
static GstPadProbeReturn
tracking_done_buf_prob (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
guint index = bin->index;
AppCtx *appCtx = bin->appCtx;
GstBuffer *buf = (GstBuffer *) info->data;
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
if (!batch_meta) {
NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
return GST_PAD_PROBE_OK;
}
GstMapInfo in_map_info;
NvBufSurface *surface = NULL;
memset (&in_map_info, 0, sizeof (in_map_info));
if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
g_print ("Error: Failed to map gst buffer\n");
}
surface = (NvBufSurface *) in_map_info.data;
NvDsMetaList * l_frame = NULL;
NvDsMetaList * l_user_meta = NULL;
NvDsUserMeta *user_meta = NULL;
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
l_frame = l_frame->next) {
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
char *src_data = (char*) malloc(surface->surfaceList[frame_meta->batch_id].dataSize);
if(src_data == NULL) {
g_print("Error: failed to malloc src_data \n");
continue;
}
cudaError_t err =cudaMemcpy((void*)src_data,
(void*)surface->surfaceList[frame_meta->batch_id].dataPtr,
surface->surfaceList[frame_meta->batch_id].dataSize,
cudaMemcpyDeviceToHost);
uint32_t frame_width = surface->surfaceList[frame_meta->batch_id].width;
uint32_t frame_height = surface->surfaceList[frame_meta->batch_id].height;
char filename[200];
sprintf(filename,"file_y_%dx%d.jpg",frame_width,frame_height);
write_jpeg_file( filename, src_data , frame_width, frame_height, 1, JCS_GRAYSCALE );
if(src_data != NULL) {
free(src_data);
src_data = NULL;
}
}
gst_buffer_unmap (buf, &in_map_info);
/*
* Output KITTI labels with tracking ID if configured to do so.
*/
write_kitti_track_output(appCtx, batch_meta);
if (appCtx->primary_bbox_generated_cb)
appCtx->primary_bbox_generated_cb (appCtx, buf, batch_meta, index);
return GST_PAD_PROBE_OK;
}
I also get cudaError_t 11
I have attached the output. Kindly let me know where I must be going wrong. I have tested the libjpeg based code and it works fine separately.
Thanks.
Which platform and DS version did you use?
Hi bcao,
I am working on Jetson Nano and with deepstream 4.0 sdk.
Thanks
You need to find the header files if make sure you had set up the environment correctly
Hi bcao,
I am working on Jetson Nano and with deepstream 4.0.1 sdk.
I am trying to use your advice given above “dump in tiler_src_pad_buffer_probe example”.
I also get seg_meta_data == NULL. Kindly let me know where I must be going wrong.
Thanks
deepstream_infer_tensor_meta_test.cpp (30.6 KB)
You need to add the probe call back to the right element(nvinfer src or the after element).