/* * Copyright (c) 2018 NVIDIA Corporation. All rights reserved. * * NVIDIA Corporation and its licensors retain all intellectual property * and proprietary rights in and to this software, related documentation * and any modifications thereto. Any use, reproduction, disclosure or * distribution of this software and related documentation without an express * license agreement from NVIDIA Corporation is strictly prohibited. * */ #include #include #include #include "gstnvdsmeta.h" #define MAX_DISPLAY_LEN 64 #define PGIE_CLASS_ID_VEHICLE 0 #define PGIE_CLASS_ID_PERSON 2 gint frame_number = 0; gchar pgie_classes_str[4][32] = { "Vehicle", "TwoWheeler", "Person", "Roadsign" }; static GstPadProbeReturn sink_bin_buf_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data) { static struct timeval last_fps_time; static struct timeval curr_fps_time; static int numbuff = 0; /* this parameter determines after how many frames fps metrics are reported */ const int FPS_DISPLAY_FREQ = 10; if ((numbuff++ % FPS_DISPLAY_FREQ) == 0) { //last fps is not recorded if (last_fps_time.tv_sec == 0 && last_fps_time.tv_usec == 0) /* first time around */ gettimeofday (&last_fps_time, NULL); else { gettimeofday (&curr_fps_time, NULL); gdouble elapsedtime = (curr_fps_time.tv_sec + curr_fps_time.tv_usec / 1000000.0) - (last_fps_time.tv_sec + last_fps_time.tv_usec / 1000000.0); g_print ("%.2f fps\n", (FPS_DISPLAY_FREQ/elapsedtime)); last_fps_time = curr_fps_time; } } return GST_PAD_PROBE_OK; } /* osd_sink_pad_buffer_probe will extract metadata received on OSD sink pad * and update params for drawing rectangle, object information etc. */ static GstPadProbeReturn osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data) { GstMeta *gst_meta = NULL; NvDsMeta *nvdsmeta = NULL; gpointer state = NULL; static GQuark _nvdsmeta_quark = 0; GstBuffer *buf = (GstBuffer *) info->data; NvDsFrameMeta *frame_meta = NULL; guint num_rects = 0, rect_index = 0, l_index = 0; NvDsObjectParams *obj_meta = NULL; guint i = 0; NvOSD_TextParams *txt_params = NULL; guint vehicle_count = 0; guint person_count = 0; if (!_nvdsmeta_quark) _nvdsmeta_quark = g_quark_from_static_string (NVDS_META_STRING); while ((gst_meta = gst_buffer_iterate_meta (buf, &state))) { if (gst_meta_api_type_has_tag (gst_meta->info->api, _nvdsmeta_quark)) { nvdsmeta = (NvDsMeta *) gst_meta; /* We are interested only in intercepting Meta of type * "NVDS_META_FRAME_INFO" as they are from our infer elements. */ if (nvdsmeta->meta_type == NVDS_META_FRAME_INFO) { frame_meta = (NvDsFrameMeta *) nvdsmeta->meta_data; if (frame_meta == NULL) { g_print ("NvDS Meta contained NULL meta \n"); return GST_PAD_PROBE_OK; } /* We reset the num_strings here as we plan to iterate through the * the detected objects and form our own strings. * The pipeline generated strings shall be discarded. */ frame_meta->num_strings = 0; num_rects = frame_meta->num_rects; /* This means we have num_rects in frame_meta->obj_params, * now lets iterate through them */ for (rect_index = 0; rect_index < num_rects; rect_index++) { /* Now using above information we need to form a text that should * be displayed on top of the bounding box, so lets form it here. */ obj_meta = (NvDsObjectParams *) & frame_meta->obj_params[rect_index]; txt_params = &(obj_meta->text_params); if (txt_params->display_text) g_free (txt_params->display_text); txt_params->display_text = g_malloc0 (MAX_DISPLAY_LEN); g_snprintf (txt_params->display_text, MAX_DISPLAY_LEN, "%s ", pgie_classes_str[obj_meta->class_id]); if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) vehicle_count++; if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) person_count++; /* Now set the offsets where the string should appear */ txt_params->x_offset = obj_meta->rect_params.left; txt_params->y_offset = obj_meta->rect_params.top - 25; /* Font , font-color and font-size */ txt_params->font_params.font_name = "Arial"; txt_params->font_params.font_size = 10; txt_params->font_params.font_color.red = 1.0; txt_params->font_params.font_color.green = 1.0; txt_params->font_params.font_color.blue = 1.0; txt_params->font_params.font_color.alpha = 1.0; /* Text background color */ txt_params->set_bg_clr = 1; txt_params->text_bg_clr.red = 0.0; txt_params->text_bg_clr.green = 0.0; txt_params->text_bg_clr.blue = 0.0; txt_params->text_bg_clr.alpha = 1.0; frame_meta->num_strings++; } } } } g_print ("Frame Number = %d Number of objects = %d " "Vehicle Count = %d Person Count = %d\n", frame_number, num_rects, vehicle_count, person_count); frame_number++; return GST_PAD_PROBE_OK; } static gboolean bus_call (GstBus * bus, GstMessage * msg, gpointer data) { GMainLoop *loop = (GMainLoop *) data; switch (GST_MESSAGE_TYPE (msg)) { case GST_MESSAGE_EOS: g_print ("End of stream\n"); g_main_loop_quit (loop); break; case GST_MESSAGE_ERROR:{ gchar *debug; GError *error; gst_message_parse_error (msg, &error, &debug); g_printerr ("ERROR from element %s: %s\n", GST_OBJECT_NAME (msg->src), error->message); if (debug) g_printerr ("Error details: %s\n", debug); g_free (debug); g_error_free (error); g_main_loop_quit (loop); break; } default: break; } return TRUE; } int main (int argc, char *argv[]) { GMainLoop *loop = NULL; GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, *decoder = NULL, *sink = NULL, *pgie = NULL, *nvvidconv = NULL, *nvosd = NULL, *filter1 = NULL, *filter2 = NULL; GstBus *bus = NULL; guint bus_watch_id; GstCaps *caps1 = NULL, *caps2 = NULL; gulong osd_probe_id = 0; gulong perf_probe_id = 0; GstPad *osd_sink_pad = NULL; GstElement *nvvidconv1 = NULL; GstElement *filter3 = NULL; GstElement *videoconvert = NULL; GstElement *filter4 = NULL; GstElement *x264enc = NULL; GstElement *qtmux = NULL; GstCaps *caps3 = NULL, *caps4 = NULL; /* Check input arguments */ if (argc != 2) { g_printerr ("Usage: %s \n", argv[0]); return -1; } /* 标准的GStreamer初始化流程 */ gst_init (&argc, &argv); loop = g_main_loop_new (NULL, FALSE); /* 创建 gstreamer elements */ /* 创建 Pipeline element 随后将于其他的elements形成一个connnection */ pipeline = gst_pipeline_new ("dstest1-pipeline"); /* Source element 用于读取file */ source = gst_element_factory_make ("filesrc", "file-source"); /* 既然输入文件的数据格式式h264视频流， * 所以需要 h264parser element */ h264parser = gst_element_factory_make ("h264parse", "h264-parser"); /* 使用 nvdec_h264 element 在GPU上实现硬件加速解码 */ decoder = gst_element_factory_make ("nvdec_h264", "nvh264-decoder"); /* 使用 nvinfer element 在解码后的数据上进行推理, * 推理中要用到的参数基于 config file */ pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine"); /* 使用 nvvidconv element 实现 NV12 to RGBA 的格式转换 */ nvvidconv = gst_element_factory_make ("nvvidconv", "nvvideo-converter"); /* 使用 nvosd element 基于RBGA的缓存来屏幕输出（由nvvidconv提供） */ nvosd = gst_element_factory_make ("nvosd", "nv-onscreendisplay"); nvvidconv1 = gst_element_factory_make("nvvidconv", "nvvideo-converter1"); videoconvert = gst_element_factory_make("videoconvert", "converter"); x264enc = gst_element_factory_make("x264enc", "h264 encoder"); qtmux = gst_element_factory_make("qtmux", "muxer"); /* 最后 filesink element 把osd屏幕输出的内容变为文件 */ sink = gst_element_factory_make ("filesink", "filesink"); /* caps filter for nvvidconv to convert NV12 to RGBA as nvosd expects input * in RGBA format */ filter1 = gst_element_factory_make ("capsfilter", "filter1"); filter2 = gst_element_factory_make ("capsfilter", "filter2"); filter3 = gst_element_factory_make ("capsfilter", "filter3"); filter4 = gst_element_factory_make ("capsfilter", "filter4"); if (!pipeline || !source || !h264parser || !decoder || !pgie // 任意一个element为flase，则逻辑非+逻辑或执行后，会运行此代码跳出程序 || !filter1 || !nvvidconv || !filter2 || !nvosd || !sink) { g_printerr ("One element could not be created. Exiting.\n"); return -1; } if (!nvvidconv1 || !x264enc || !qtmux || !filter3 || !filter4) { g_printerr("one element could not be created. Exiting.\n"); return -1; } /* 我们把 input filename 给到 source element */ g_object_set (G_OBJECT (source), "location", argv[1], NULL); /* 我们把 最后sink element output的文件定义为 out_test1.mp4 */ g_object_set(G_OBJECT(sink), "location", "out_test2.mp4", NULL); /* 设置nvinfer element 的所有必要属性, * 必备的config文件路径 : */ g_object_set (G_OBJECT (pgie), "config-file-path", "dstest1_pgie_config.txt", NULL); /* 我们在这里设置了 osd 属性 */ g_object_set (G_OBJECT (nvosd), "font-size", 15, NULL); /* 我们添加了一个消息处理程序（message handler） */ bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); gst_object_unref (bus); /* //////////////////////////////////////////////////////////////////// // <1> 建立 pipeline 通过添加所需的 elements //////////////////////////////////////////////////////////////////// // TODO1 // Hints: // 1> Some elements are missing from the pipeline. You can refer to the elements defined above to see which ones aren't currently incorporated. 2> Think about the kind of data stream we're handling */ gst_bin_add_many (GST_BIN (pipeline), source, h264parser, decoder, pgie, filter1, nvvidconv, filter2, nvosd, nvvidconv1, filter3, videoconvert, filter4, x264enc, qtmux, sink, NULL); caps1 = gst_caps_from_string ("video/x-raw(memory:NVMM), format=NV12"); g_object_set (G_OBJECT (filter1), "caps", caps1, NULL); gst_caps_unref (caps1); caps2 = gst_caps_from_string ("video/x-raw(memory:NVMM), format=RGBA"); g_object_set (G_OBJECT (filter2), "caps", caps2, NULL); gst_caps_unref (caps2); caps3 = gst_caps_from_string("video/x-raw, format=RGBA"); g_object_set(G_OBJECT(filter3), "caps", caps3, NULL); gst_caps_unref(caps3); caps4 = gst_caps_from_string("video/x-raw, format=NV12"); g_object_set(G_OBJECT(filter4), "caps", caps4, NULL); gst_caps_unref(caps4); /* 我们随后把所有的 elements 链接到一起 */ /* file-source -> h264-parser -> nvh264-decoder -> * nvinfer -> filter1 -> nvvidconv -> filter2 -> nvosd -> video-renderer */ gst_element_link_many (source, h264parser, decoder, pgie, filter1, nvvidconv, filter2, nvosd, nvvidconv1, filter3, videoconvert, filter4, x264enc, qtmux, sink, NULL); /* 添加 probe来获取生成的 metadata 信息, 我们在 osd element 的sink pad上添加 probe * 因为那时, the buffer 中已经有了所有的metadata. */ osd_sink_pad = gst_element_get_static_pad (nvosd, "sink"); if (!osd_sink_pad) g_print ("Unable to get sink pad\n"); else { osd_probe_id = gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, osd_sink_pad_buffer_probe, NULL, NULL); /* 如果要实现Performance Analysis Exercise，可使用如下语句 */ // perf_probe_id = gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, // sink_bin_buf_probe, NULL, NULL); } /* //////////////////////////////////////////////////////////////////// // <2> 将pipline状态设置为实际呈现传入数据 //////////////////////////////////////////////////////////////////// // TODO2 // Hints: // 1> 查看 https://gstreamer.freedesktop.org/documentation/plugin-development/basics/states.html 以获取所有可能的状态列表 2> 我们希望选择一种状态，它允许我们一路发送数据 */ g_print ("Now playing: %s\n", argv[1]); gst_element_set_state (pipeline, GST_STATE_PLAYING // 存在4中状态，"NULL"，"READY"，"PAUSED"和"PLAYING"，此处设置为PLAYING ); /* 执行main loop直到 pipeline 遇到错误或者 EOS手动停止 */ g_print ("Running...\n"); g_main_loop_run (loop); /* 跳出main loop, clean up nicely */ g_print ("Returned, stopping playback\n"); /* //////////////////////////////////////////////////////////////////// // <3> 将pipeline状态设置为关闭设备 and clean up //////////////////////////////////////////////////////////////////// // TODO3 // Hints: // 1> See https://gstreamer.freedesktop.org/documentation/plugin-development/basics/states.html to find the list of possible states 2> We want to choose the state that clears our resources. */ gst_element_set_state (pipeline, GST_STATE_NULL // 默认的初始状态，在此状态下，它没有分配任何运行时资源 ); g_print ("Deleting pipeline\n"); gst_object_unref (GST_OBJECT (pipeline)); g_source_remove (bus_watch_id); g_main_loop_unref (loop); return 0; }