Hi~
Can I use deepstream yolo to process mp4 video file instead of h264 format video file?
And how can I parse the detected bounding box’ coordinates and label and confidence,and also the frame sequence inside input video file?
thanks~
Hi,
You can change the input element from filesrc->h264parse->h264dec to uridecodebin which will give you the decoded frames from your mp4 container.
You can read the bounding box coordinates from the metadata through the probe here - https://github.com/NVIDIA-AI-IOT/deepstream_reference_apps/blob/master/yolo/apps/deepstream-yolo/deepstream-yolo-app.cpp#L82
Here is my code trying to save the sink as a mp4 video file,and it worked for me.
int
main (int argc, char *argv[])
{
GMainLoop *loop = NULL;
GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, *decoder =
NULL, *sink = NULL, *nvvidconv = NULL, *nvosd = NULL, *filter1 =
NULL, *filter2 = NULL, *yolo = NULL;
GstBus *bus = NULL;
guint bus_watch_id;
GstCaps *caps1 = NULL, *caps2 = NULL;
gulong osd_probe_id = 0;
GstPad *osd_sink_pad = NULL;
// ADD begin
GstElement *nvvidconv1 = NULL,
*filter3 = NULL,
*videoconvert = NULL,
*filter4 = NULL,
*x264enc = NULL,
*qtmux = NULL;
GstCaps *caps3 = NULL, *caps4 = NULL;
// ADD end
/* Check input arguments */
if (argc != 5) {
g_printerr
("Usage: %s <Platform-Telsa/Tegra> <H264 filename> <yolo-plugin config file> \n",
argv[0]);
return -1;
}
double start_time = what_time_is_it_now();
time_t t;
t = time(NULL);
std::cout << "Start Time now : " << ctime(&t) << std::endl;
/* Standard GStreamer initialization */
gst_init (&argc, &argv);
loop = g_main_loop_new (NULL, FALSE);
/* Create gstreamer elements */
/* Create Pipeline element that will form a connection of other elements */
pipeline = gst_pipeline_new ("ds-yolo-pipeline");
/* Source element for reading from the file */
source = gst_element_factory_make ("filesrc", "file-source");
/* Since the data format in the input file is elementary h264 stream,
* we need a h264parser */
h264parser = gst_element_factory_make ("h264parse", "h264-parser");
/* Use nvdec_h264/omxh264dec for hardware accelerated decode on GPU */
if (!g_strcmp0 ("Tesla", argv[1])) {
decoder = gst_element_factory_make ("nvdec_h264", "nvh264-decoder");
} else if (!g_strcmp0 ("Tegra", argv[1])) {
decoder = gst_element_factory_make ("omxh264dec", "openmax-decoder");
} else {
g_printerr ("Incorrect platform. Choose between Telsa/Tegra. Exiting.\n");
return -1;
}
/* Use convertor to convert from NV12 to RGBA as required by nvosd and yolo plugins */
nvvidconv = gst_element_factory_make ("nvvidconv", "nvvideo-converter");
/* Use yolo to run inference instead of pgie */
yolo = gst_element_factory_make ("nvyolo", "yolo-inference-engine");
/* Create OSD to draw on the converted RGBA buffer */
nvosd = gst_element_factory_make ("nvosd", "nv-onscreendisplay");
// ADD begin
/* Finally render the osd output */
nvvidconv1 = gst_element_factory_make ("nvvidconv", "nvvideo-converter1");
videoconvert = gst_element_factory_make ("videoconvert", "converter");
x264enc = gst_element_factory_make ("x264enc", "h264 encoder");
qtmux = gst_element_factory_make ("qtmux", "muxer");
// ADD end
/* Finally render the osd output */
if (!g_strcmp0 ("Tesla", argv[1])) {
// sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer");
sink = gst_element_factory_make ("filesink", "filesink");
} else if (!g_strcmp0 ("Tegra", argv[1])) {
sink = gst_element_factory_make ("nvoverlaysink", "nvvideo-renderer");
} else {
g_printerr ("Incorrect platform. Choose between Telsa/Tegra. Exiting.\n");
return -1;
}
/* caps filter for nvvidconv to convert NV12 to RGBA as nvosd expects input
* in RGBA format */
filter1 = gst_element_factory_make ("capsfilter", "filter1");
filter2 = gst_element_factory_make ("capsfilter", "filter2");
filter3 = gst_element_factory_make ("capsfilter", "filter3");
filter4 = gst_element_factory_make ("capsfilter", "filter4");
if (!pipeline || !source || !h264parser || !decoder || !filter1 || !nvvidconv
|| !filter2 || !nvosd || !sink || !yolo) {
g_printerr ("One element could not be created. Exiting.\n");
return -1;
}
/* we set the input filename to the source element */
g_object_set (G_OBJECT (source), "location", argv[2], NULL);
g_object_set(G_OBJECT(sink), "location", argv[4], NULL);
g_object_set (G_OBJECT (yolo), "config-file-path", argv[3], NULL);
/* we set the osd properties here */
g_object_set (G_OBJECT (nvosd), "font-size", 15, NULL);
/* we add a message handler */
bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
gst_object_unref (bus);
/* Set up the pipeline */
/* we add all elements into the pipeline */
// gst_bin_add_many (GST_BIN (pipeline), source, h264parser, decoder, filter1,
// nvvidconv, filter2, yolo, nvosd, sink, NULL);
gst_bin_add_many (GST_BIN (pipeline), source, h264parser, decoder, filter1,
nvvidconv, filter2, yolo, nvosd, nvvidconv1, sink, filter3, videoconvert,
filter4, x264enc, qtmux, NULL);
caps1 = gst_caps_from_string ("video/x-raw(memory:NVMM), format=NV12");
g_object_set (G_OBJECT (filter1), "caps", caps1, NULL);
gst_caps_unref (caps1);
caps2 = gst_caps_from_string ("video/x-raw(memory:NVMM), format=RGBA");
g_object_set (G_OBJECT (filter2), "caps", caps2, NULL);
gst_caps_unref (caps2);
// ADD begin
caps3 = gst_caps_from_string ("video/x-raw, format=RGBA");
g_object_set (G_OBJECT (filter3), "caps", caps3, NULL);
gst_caps_unref (caps3);
caps4 = gst_caps_from_string ("video/x-raw, format=NV12");
g_object_set (G_OBJECT (filter4), "caps", caps4, NULL);
gst_caps_unref (caps4);
// ADD end
/* we link the elements together */
/* file-source -> h264-parser -> nvh264-decoder ->
* filter1 -> nvvidconv -> filter2 -> yolo -> nvosd -> video-renderer */
// gst_element_link_many (source, h264parser, decoder, filter1, nvvidconv,
// filter2, yolo, nvosd, sink, NULL);
gst_element_link_many (source, h264parser, decoder, filter1, nvvidconv, filter2, yolo,
nvosd, nvvidconv1, filter3, videoconvert, filter4, x264enc, qtmux, sink, NULL);
/* Lets add probe to get informed of the meta data generated, we add probe to
* the sink pad of the osd element, since by that time, the buffer would have
* had got all the metadata. */
osd_sink_pad = gst_element_get_static_pad (nvosd, "sink");
if (!osd_sink_pad)
g_print ("Unable to get sink pad\n");
else
osd_probe_id = gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER,
osd_sink_pad_buffer_probe, NULL, NULL);
/* Set the pipeline to "playing" state */
g_print ("Now playing: %s\n", argv[2]);
gst_element_set_state (pipeline, GST_STATE_PLAYING);
/* Wait till pipeline encounters an error or EOS */
g_print ("Running...\n");
g_main_loop_run (loop);
/* Out of the main loop, clean up nicely */
g_print ("Returned, stopping playback\n");
gst_element_set_state (pipeline, GST_STATE_NULL);
g_print ("Deleting pipeline\n");
gst_object_unref (GST_OBJECT (pipeline));
g_source_remove (bus_watch_id);
g_main_loop_unref (loop);
double end_time = what_time_is_it_now();
g_print("\nTotal processing time : %d", (end_time - start_time));
time_t t2;
t2 = time(NULL);
std::cout << "End Time now : " << ctime(&t2) << std::endl;
return 0;
}
But I still don’t know how to read a mp4 file?
I also tried to adjust my code according to deepstream-test1:
new adding code for reading mp4 file begins with “// new ADD begin” ends with “new ADD end”
// new ADD begin
static void
cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data)
{
GstCaps *caps = gst_pad_query_caps (decoder_src_pad, NULL);
const GstStructure *str = gst_caps_get_structure (caps, 0);
const gchar *name = gst_structure_get_name (str);
GstElement *source_bin = (GstElement *) data;
GstCapsFeatures *features = gst_caps_get_features (caps, 0);
/* Need to check if the pad created by the decodebin is for video and not
* audio. */
if (!strncmp (name, "video", 5)) {
/* Link the decodebin pad only if decodebin has picked nvidia
* decoder plugin nvdec_*. We do this by checking if the pad caps contain
* NVMM memory features. */
if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) {
/* Get the source bin ghost pad */
GstPad *bin_ghost_pad = gst_element_get_static_pad (source_bin, "src");
if (!gst_ghost_pad_set_target (GST_GHOST_PAD (bin_ghost_pad),
decoder_src_pad)) {
g_printerr ("Failed to link decoder src pad to source bin ghost pad\n");
}
gst_object_unref (bin_ghost_pad);
} else {
g_printerr ("Error: Decodebin did not pick nvidia decoder plugin.\n");
}
}
}
static GstElement *
create_source_bin (guint index, gchar * uri)
{
GstElement *bin = NULL, *uri_decode_bin = NULL;
gchar bin_name[16] = { };
g_snprintf (bin_name, 15, "source-bin-%02d", index);
/* Create a source GstBin to abstract this bin's content from the rest of the
* pipeline */
bin = gst_bin_new (bin_name);
/* Source element for reading from the uri.
* We will use decodebin and let it figure out the container format of the
* stream and the codec and plug the appropriate demux and decode plugins. */
uri_decode_bin = gst_element_factory_make ("uridecodebin", "uri-decode-bin");
if (!bin || !uri_decode_bin) {
g_printerr ("One element in source bin could not be created.\n");
return NULL;
}
/* We set the input uri to the source element */
g_object_set (G_OBJECT (uri_decode_bin), "uri", uri, NULL);
/* Connect to the "pad-added" signal of the decodebin which generates a
* callback once a new pad for raw data has beed created by the decodebin */
g_signal_connect (G_OBJECT (uri_decode_bin), "pad-added",
G_CALLBACK (cb_newpad), bin);
gst_bin_add (GST_BIN (bin), uri_decode_bin);
/* We need to create a ghost pad for the source bin which will act as a proxy
* for the video decoder src pad. The ghost pad will not have a target right
* now. Once the decode bin creates the video decoder and generates the
* cb_newpad callback, we will set the ghost pad target to the video decoder
* src pad. */
if (!gst_element_add_pad (bin, gst_ghost_pad_new_no_target ("src",
GST_PAD_SRC))) {
g_printerr ("Failed to add ghost pad in source bin\n");
return NULL;
}
return bin;
}
// new ADD end
int
main (int argc, char *argv[])
{
GMainLoop *loop = NULL;
GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, *decoder =
NULL, *sink = NULL, *nvvidconv = NULL, *nvosd = NULL, *filter1 =
NULL, *filter2 = NULL, *yolo = NULL;
GstBus *bus = NULL;
guint bus_watch_id;
GstCaps *caps1 = NULL, *caps2 = NULL;
gulong osd_probe_id = 0;
GstPad *osd_sink_pad = NULL;
// ADD begin
GstElement *nvvidconv1 = NULL,
*filter3 = NULL,
*videoconvert = NULL,
*filter4 = NULL,
*x264enc = NULL,
*qtmux = NULL;
GstCaps *caps3 = NULL, *caps4 = NULL;
// ADD end
// new ADD begin
GstElement *streammux = NULL;
// new ADD end
/* Check input arguments */
if (argc != 5) {
g_printerr
("Usage: %s <Platform-Telsa/Tegra> <H264 filename> <yolo-plugin config file> \n",
argv[0]);
return -1;
}
double start_time = what_time_is_it_now();
time_t t;
t = time(NULL);
std::cout << "Start Time now : " << ctime(&t) << std::endl;
/* Standard GStreamer initialization */
gst_init (&argc, &argv);
loop = g_main_loop_new (NULL, FALSE);
/* Create gstreamer elements */
/* Create Pipeline element that will form a connection of other elements */
pipeline = gst_pipeline_new ("ds-yolo-pipeline");
// new ADD begin
streammux = gst_element_factory_make ("nvstreammux", "stream-muxer");
if (!pipeline || !streammux) {
g_printerr ("One element could not be created. Exiting.\n");
return -1;
}
gst_bin_add (GST_BIN (pipeline), streammux);
GstPad *sinkpad, *srcpad;
gchar pad_name[16] = { };
GstElement *source_bin = create_source_bin (0, argv[2]);
if (!source_bin) {
g_printerr ("Failed to create source bin. Exiting.\n");
return -1;
}
gst_bin_add (GST_BIN (pipeline), source_bin);
sinkpad = gst_element_get_request_pad (streammux, pad_name);
if (!sinkpad) {
g_printerr ("Streammux request sink pad failed. Exiting.\n");
return -1;
}
srcpad = gst_element_get_static_pad (source_bin, "src");
if (!srcpad) {
g_printerr ("Failed to get src pad of source bin. Exiting.\n");
return -1;
}
if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) {
g_printerr ("Failed to link source bin to stream muxer. Exiting.\n");
return -1;
}
gst_object_unref (srcpad);
gst_object_unref (sinkpad);
new ADD end
/* Source element for reading from the file */
// source = gst_element_factory_make ("filesrc", "file-source");
/* Since the data format in the input file is elementary h264 stream,
* we need a h264parser */
h264parser = gst_element_factory_make ("h264parse", "h264-parser");
/* Use nvdec_h264/omxh264dec for hardware accelerated decode on GPU */
if (!g_strcmp0 ("Tesla", argv[1])) {
decoder = gst_element_factory_make ("nvdec_h264", "nvh264-decoder");
} else if (!g_strcmp0 ("Tegra", argv[1])) {
decoder = gst_element_factory_make ("omxh264dec", "openmax-decoder");
} else {
g_printerr ("Incorrect platform. Choose between Telsa/Tegra. Exiting.\n");
return -1;
}
/* Use convertor to convert from NV12 to RGBA as required by nvosd and yolo plugins */
nvvidconv = gst_element_factory_make ("nvvidconv", "nvvideo-converter");
/* Use yolo to run inference instead of pgie */
yolo = gst_element_factory_make ("nvyolo", "yolo-inference-engine");
/* Create OSD to draw on the converted RGBA buffer */
nvosd = gst_element_factory_make ("nvosd", "nv-onscreendisplay");
// ADD begin
/* Finally render the osd output */
nvvidconv1 = gst_element_factory_make ("nvvidconv", "nvvideo-converter1");
videoconvert = gst_element_factory_make ("videoconvert", "converter");
x264enc = gst_element_factory_make ("x264enc", "h264 encoder");
qtmux = gst_element_factory_make ("qtmux", "muxer");
// ADD end
// new ADD begin
g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height",
MUXER_OUTPUT_HEIGHT, "batch-size", 1,
"batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL);
// new ADD end
/* Finally render the osd output */
if (!g_strcmp0 ("Tesla", argv[1])) {
// sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer");
sink = gst_element_factory_make ("filesink", "filesink");
} else if (!g_strcmp0 ("Tegra", argv[1])) {
sink = gst_element_factory_make ("nvoverlaysink", "nvvideo-renderer");
} else {
g_printerr ("Incorrect platform. Choose between Telsa/Tegra. Exiting.\n");
return -1;
}
/* caps filter for nvvidconv to convert NV12 to RGBA as nvosd expects input
* in RGBA format */
filter1 = gst_element_factory_make ("capsfilter", "filter1");
filter2 = gst_element_factory_make ("capsfilter", "filter2");
filter3 = gst_element_factory_make ("capsfilter", "filter3");
filter4 = gst_element_factory_make ("capsfilter", "filter4");
if (!pipeline || !h264parser || !decoder || !filter1 || !nvvidconv
|| !filter2 || !nvosd || !sink || !yolo) {
g_printerr ("One element could not be created. Exiting.\n");
return -1;
}
/* we set the input filename to the source element */
// g_object_set (G_OBJECT (source), "location", argv[2], NULL);
g_object_set(G_OBJECT(sink), "location", argv[4], NULL);
g_object_set (G_OBJECT (yolo), "config-file-path", argv[3], NULL);
/* we set the osd properties here */
g_object_set (G_OBJECT (nvosd), "font-size", 15, NULL);
/* we add a message handler */
bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
gst_object_unref (bus);
/* Set up the pipeline */
/* we add all elements into the pipeline */
// gst_bin_add_many (GST_BIN (pipeline), source, h264parser, decoder, filter1,
// nvvidconv, filter2, yolo, nvosd, sink, NULL);
gst_bin_add_many (GST_BIN (pipeline), h264parser, decoder, filter1,
nvvidconv, filter2, yolo, nvosd, nvvidconv1, sink, filter3, videoconvert,
filter4, x264enc, qtmux, NULL);
caps1 = gst_caps_from_string ("video/x-raw(memory:NVMM), format=NV12");
g_object_set (G_OBJECT (filter1), "caps", caps1, NULL);
gst_caps_unref (caps1);
caps2 = gst_caps_from_string ("video/x-raw(memory:NVMM), format=RGBA");
g_object_set (G_OBJECT (filter2), "caps", caps2, NULL);
gst_caps_unref (caps2);
// ADD begin
caps3 = gst_caps_from_string ("video/x-raw, format=RGBA");
g_object_set (G_OBJECT (filter3), "caps", caps3, NULL);
gst_caps_unref (caps3);
caps4 = gst_caps_from_string ("video/x-raw, format=NV12");
g_object_set (G_OBJECT (filter4), "caps", caps4, NULL);
gst_caps_unref (caps4);
// ADD end
/* we link the elements together */
/* file-source -> h264-parser -> nvh264-decoder ->
* filter1 -> nvvidconv -> filter2 -> yolo -> nvosd -> video-renderer */
// gst_element_link_many (source, h264parser, decoder, filter1, nvvidconv,
// filter2, yolo, nvosd, sink, NULL);
gst_element_link_many (streammux, h264parser, decoder, filter1, nvvidconv, filter2, yolo,
nvosd, nvvidconv1, filter3, videoconvert, filter4, x264enc, qtmux, sink, NULL);
/* Lets add probe to get informed of the meta data generated, we add probe to
* the sink pad of the osd element, since by that time, the buffer would have
* had got all the metadata. */
osd_sink_pad = gst_element_get_static_pad (nvosd, "sink");
if (!osd_sink_pad)
g_print ("Unable to get sink pad\n");
else
osd_probe_id = gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER,
osd_sink_pad_buffer_probe, NULL, NULL);
/* Set the pipeline to "playing" state */
g_print ("Now playing: %s\n", argv[2]);
gst_element_set_state (pipeline, GST_STATE_PLAYING);
/* Wait till pipeline encounters an error or EOS */
g_print ("Running...\n");
g_main_loop_run (loop);
/* Out of the main loop, clean up nicely */
g_print ("Returned, stopping playback\n");
gst_element_set_state (pipeline, GST_STATE_NULL);
g_print ("Deleting pipeline\n");
gst_object_unref (GST_OBJECT (pipeline));
g_source_remove (bus_watch_id);
g_main_loop_unref (loop);
double end_time = what_time_is_it_now();
g_print("\nTotal processing time : %d", (end_time - start_time));
time_t t2;
t2 = time(NULL);
std::cout << "End Time now : " << ctime(&t2) << std::endl;
return 0;
}
But I failed to read mp4 file.Here is my error message.
Streammux request sink pad failed. Exiting.
Can you provide me with more details about how to read mp4 video file from deepstream-yolo-app.cpp?Thank you very much
You seem to be passing an empty string in place of pad_name? You seem to be referring to deepstream-test3 app and you forgot to load the pad_name like this
g_snprintf (pad_name, 15, “sink_%u”, i);
You can also use deepstream-app to perform yolo inference. You can have a look at the sample here - https://github.com/NVIDIA-AI-IOT/deepstream_reference_apps/tree/master/yolo/samples/objectDetector_YoloV3
Hi
After adding code
g_snprintf (pad_name, 15, "sink_%u", i);
when I run the command:
./deepstream-yolo-app Tesla sample_720p.mp4 ../../../config/yolov3.txt output.mp4
I got this error:
Running...
(deepstream-yolo-app:41186): GStreamer-CRITICAL **: _gst_util_uint64_scale_int: assertion 'num >= 0' failed
ERROR from element nvh264-decoder: No valid frames decoded before end of stream
Error: No valid frames decoded before end of stream
Returned, stopping playback
Yolo Plugin Perf Summary
Batch Size : 1
PreProcess : -nan ms Inference : -nan ms PostProcess : -nan ms Total : -nan ms per Image
Deleting pipeline
It seemed that didn’t work for me when being referring to deepstream-test3 app in order to read mp4 file.
Howevr when I run this command:
./deepstream-yolo-app Tesla sample_720p.h264 ../../../config/yolov3.txt output.mp4
It seemed to work fine.And save the result as a mp4 file with bounding box attached to detected objs.
(objectDetector_YoloV3 demo I will update laterly to see if I can use in my project.But not take into account now…)
In this way, I think I already change the input element from filesrc->h264parse->h264dec to uridecodebin so as to read mp4 file.Here is my code:
// new ADD begin
static void
cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data)
{
GstCaps *caps = gst_pad_query_caps (decoder_src_pad, NULL);
const GstStructure *str = gst_caps_get_structure (caps, 0);
const gchar *name = gst_structure_get_name (str);
GstElement *source_bin = (GstElement *) data;
GstCapsFeatures *features = gst_caps_get_features (caps, 0);
/* Need to check if the pad created by the decodebin is for video and not
* audio. */
if (!strncmp (name, "video", 5)) {
/* Link the decodebin pad only if decodebin has picked nvidia
* decoder plugin nvdec_*. We do this by checking if the pad caps contain
* NVMM memory features. */
if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) {
/* Get the source bin ghost pad */
GstPad *bin_ghost_pad = gst_element_get_static_pad (source_bin, "src");
if (!gst_ghost_pad_set_target (GST_GHOST_PAD (bin_ghost_pad),
decoder_src_pad)) {
g_printerr ("Failed to link decoder src pad to source bin ghost pad\n");
}
gst_object_unref (bin_ghost_pad);
} else {
g_printerr ("Error: Decodebin did not pick nvidia decoder plugin.\n");
}
}
}
static GstElement *
create_source_bin (guint index, gchar * uri)
{
GstElement *bin = NULL, *uri_decode_bin = NULL;
gchar bin_name[16] = { };
g_snprintf (bin_name, 15, "source-bin-%02d", index);
/* Create a source GstBin to abstract this bin's content from the rest of the
* pipeline */
bin = gst_bin_new (bin_name);
/* Source element for reading from the uri.
* We will use decodebin and let it figure out the container format of the
* stream and the codec and plug the appropriate demux and decode plugins. */
uri_decode_bin = gst_element_factory_make ("uridecodebin", "uri-decode-bin");
if (!bin || !uri_decode_bin) {
g_printerr ("One element in source bin could not be created.\n");
return NULL;
}
/* We set the input uri to the source element */
g_object_set (G_OBJECT (uri_decode_bin), "uri", uri, NULL);
/* Connect to the "pad-added" signal of the decodebin which generates a
* callback once a new pad for raw data has beed created by the decodebin */
g_signal_connect (G_OBJECT (uri_decode_bin), "pad-added",
G_CALLBACK (cb_newpad), bin);
gst_bin_add (GST_BIN (bin), uri_decode_bin);
/* We need to create a ghost pad for the source bin which will act as a proxy
* for the video decoder src pad. The ghost pad will not have a target right
* now. Once the decode bin creates the video decoder and generates the
* cb_newpad callback, we will set the ghost pad target to the video decoder
* src pad. */
if (!gst_element_add_pad (bin, gst_ghost_pad_new_no_target ("src",
GST_PAD_SRC))) {
g_printerr ("Failed to add ghost pad in source bin\n");
return NULL;
}
return bin;
}
// new ADD end
int
main (int argc, char *argv[])
{
GMainLoop *loop = NULL;
GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, *decoder =
NULL, *sink = NULL, *nvvidconv = NULL, *nvosd = NULL, *filter1 =
NULL, *filter2 = NULL, *yolo = NULL;
GstBus *bus = NULL;
guint bus_watch_id;
GstCaps *caps1 = NULL, *caps2 = NULL;
gulong osd_probe_id = 0;
GstPad *osd_sink_pad = NULL;
// ADD begin
GstElement *nvvidconv1 = NULL,
*filter3 = NULL,
*videoconvert = NULL,
*filter4 = NULL,
*x264enc = NULL,
*qtmux = NULL;
GstCaps *caps3 = NULL, *caps4 = NULL;
// ADD end
// new ADD begin
GstElement *streammux = NULL;
// new ADD end
/* Check input arguments */
if (argc != 5) {
g_printerr
("Usage: %s <Platform-Telsa/Tegra> <H264 filename> <yolo-plugin config file> \n",
argv[0]);
return -1;
}
double start_time = what_time_is_it_now();
time_t t;
t = time(NULL);
std::cout << "Start Time now : " << ctime(&t) << std::endl;
/* Standard GStreamer initialization */
gst_init (&argc, &argv);
loop = g_main_loop_new (NULL, FALSE);
/* Create gstreamer elements */
/* Create Pipeline element that will form a connection of other elements */
pipeline = gst_pipeline_new ("ds-yolo-pipeline");
// new ADD begin
streammux = gst_element_factory_make ("nvstreammux", "stream-muxer");
if (!pipeline || !streammux) {
g_printerr ("One element could not be created. Exiting.\n");
return -1;
}
gst_bin_add (GST_BIN (pipeline), streammux);
GstPad *sinkpad, *srcpad;
gchar pad_name[16] = { };
GstElement *source_bin = create_source_bin (0, argv[2]);
if (!source_bin) {
g_printerr ("Failed to create source bin. Exiting.\n");
return -1;
}
gst_bin_add (GST_BIN (pipeline), source_bin);
g_snprintf (pad_name, 15, "sink_%u", 0);
sinkpad = gst_element_get_request_pad (streammux, pad_name);
if (!sinkpad) {
g_printerr ("Streammux request sink pad failed. Exiting.\n");
return -1;
}
srcpad = gst_element_get_static_pad (source_bin, "src");
if (!srcpad) {
g_printerr ("Failed to get src pad of source bin. Exiting.\n");
return -1;
}
if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) {
g_printerr ("Failed to link source bin to stream muxer. Exiting.\n");
return -1;
}
gst_object_unref (srcpad);
gst_object_unref (sinkpad);
new ADD end
/* Source element for reading from the file */
// source = gst_element_factory_make ("filesrc", "file-source");
/* Since the data format in the input file is elementary h264 stream,
* we need a h264parser */
h264parser = gst_element_factory_make ("h264parse", "h264-parser");
/* Use nvdec_h264/omxh264dec for hardware accelerated decode on GPU */
if (!g_strcmp0 ("Tesla", argv[1])) {
decoder = gst_element_factory_make ("nvdec_h264", "nvh264-decoder");
} else if (!g_strcmp0 ("Tegra", argv[1])) {
decoder = gst_element_factory_make ("omxh264dec", "openmax-decoder");
} else {
g_printerr ("Incorrect platform. Choose between Telsa/Tegra. Exiting.\n");
return -1;
}
/* Use convertor to convert from NV12 to RGBA as required by nvosd and yolo plugins */
nvvidconv = gst_element_factory_make ("nvvidconv", "nvvideo-converter");
/* Use yolo to run inference instead of pgie */
yolo = gst_element_factory_make ("nvyolo", "yolo-inference-engine");
/* Create OSD to draw on the converted RGBA buffer */
nvosd = gst_element_factory_make ("nvosd", "nv-onscreendisplay");
// ADD begin
/* Finally render the osd output */
nvvidconv1 = gst_element_factory_make ("nvvidconv", "nvvideo-converter1");
videoconvert = gst_element_factory_make ("videoconvert", "converter");
x264enc = gst_element_factory_make ("x264enc", "h264 encoder");
qtmux = gst_element_factory_make ("qtmux", "muxer");
// ADD end
// new ADD begin
g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height",
MUXER_OUTPUT_HEIGHT, "batch-size", 1,
"batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL);
// new ADD end
/* Finally render the osd output */
if (!g_strcmp0 ("Tesla", argv[1])) {
// sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer");
sink = gst_element_factory_make ("filesink", "filesink");
} else if (!g_strcmp0 ("Tegra", argv[1])) {
sink = gst_element_factory_make ("nvoverlaysink", "nvvideo-renderer");
} else {
g_printerr ("Incorrect platform. Choose between Telsa/Tegra. Exiting.\n");
return -1;
}
/* caps filter for nvvidconv to convert NV12 to RGBA as nvosd expects input
* in RGBA format */
filter1 = gst_element_factory_make ("capsfilter", "filter1");
filter2 = gst_element_factory_make ("capsfilter", "filter2");
filter3 = gst_element_factory_make ("capsfilter", "filter3");
filter4 = gst_element_factory_make ("capsfilter", "filter4");
if (!pipeline || !h264parser || !decoder || !filter1 || !nvvidconv
|| !filter2 || !nvosd || !sink || !yolo) {
g_printerr ("One element could not be created. Exiting.\n");
return -1;
}
/* we set the input filename to the source element */
// g_object_set (G_OBJECT (source), "location", argv[2], NULL);
g_object_set(G_OBJECT(sink), "location", argv[4], NULL);
g_object_set (G_OBJECT (yolo), "config-file-path", argv[3], NULL);
/* we set the osd properties here */
g_object_set (G_OBJECT (nvosd), "font-size", 15, NULL);
/* we add a message handler */
bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
gst_object_unref (bus);
/* Set up the pipeline */
/* we add all elements into the pipeline */
// gst_bin_add_many (GST_BIN (pipeline), source, h264parser, decoder, filter1,
// nvvidconv, filter2, yolo, nvosd, sink, NULL);
gst_bin_add_many (GST_BIN (pipeline), h264parser, decoder, filter1,
nvvidconv, filter2, yolo, nvosd, nvvidconv1, sink, filter3, videoconvert,
filter4, x264enc, qtmux, NULL);
caps1 = gst_caps_from_string ("video/x-raw(memory:NVMM), format=NV12");
g_object_set (G_OBJECT (filter1), "caps", caps1, NULL);
gst_caps_unref (caps1);
caps2 = gst_caps_from_string ("video/x-raw(memory:NVMM), format=RGBA");
g_object_set (G_OBJECT (filter2), "caps", caps2, NULL);
gst_caps_unref (caps2);
// ADD begin
caps3 = gst_caps_from_string ("video/x-raw, format=RGBA");
g_object_set (G_OBJECT (filter3), "caps", caps3, NULL);
gst_caps_unref (caps3);
caps4 = gst_caps_from_string ("video/x-raw, format=NV12");
g_object_set (G_OBJECT (filter4), "caps", caps4, NULL);
gst_caps_unref (caps4);
// ADD end
/* we link the elements together */
/* file-source -> h264-parser -> nvh264-decoder ->
* filter1 -> nvvidconv -> filter2 -> yolo -> nvosd -> video-renderer */
// gst_element_link_many (source, h264parser, decoder, filter1, nvvidconv,
// filter2, yolo, nvosd, sink, NULL);
gst_element_link_many (streammux, h264parser, decoder, filter1, nvvidconv, filter2, yolo,
nvosd, nvvidconv1, filter3, videoconvert, filter4, x264enc, qtmux, sink, NULL);
/* Lets add probe to get informed of the meta data generated, we add probe to
* the sink pad of the osd element, since by that time, the buffer would have
* had got all the metadata. */
osd_sink_pad = gst_element_get_static_pad (nvosd, "sink");
if (!osd_sink_pad)
g_print ("Unable to get sink pad\n");
else
osd_probe_id = gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER,
osd_sink_pad_buffer_probe, NULL, NULL);
/* Set the pipeline to "playing" state */
g_print ("Now playing: %s\n", argv[2]);
gst_element_set_state (pipeline, GST_STATE_PLAYING);
/* Wait till pipeline encounters an error or EOS */
g_print ("Running...\n");
g_main_loop_run (loop);
/* Out of the main loop, clean up nicely */
g_print ("Returned, stopping playback\n");
gst_element_set_state (pipeline, GST_STATE_NULL);
g_print ("Deleting pipeline\n");
gst_object_unref (GST_OBJECT (pipeline));
g_source_remove (bus_watch_id);
g_main_loop_unref (loop);
double end_time = what_time_is_it_now();
g_print("\nTotal processing time : %d", (end_time - start_time));
time_t t2;
t2 = time(NULL);
std::cout << "End Time now : " << ctime(&t2) << std::endl;
return 0;
}
Could you help me figure out this bug?How can I read mp4 file even several mp4 file from deepstream-yolo-app.cpp being reference to deepstream-test3-app?Do I miss some codes which could decode the input mp4 video file so that it can be pushed to stream?
Thanks so much
Hey~any updates~~~~~~~~~~~~
You seem to have a h264parser and decoder after the streammux which is not needed since uridecodebin already does this for you. Can you remove those elements from the pipeline and try again ?
The reason i suggested to use objectDetector_YoloV3 sample is because it already has these features implemented.
WOW!!!
It does work!
Thank you soooooooo much!!!
I could read a small mp4 file(size:800x480 fps:50 mem:59M) and did process successfully.However when I directly input the raw mp4 file without any compression,which is 2560x1536, 50 fps and 512M,it seemed fail to process again.I got this error message:
Loading TRT Engine...
Loading Complete!
Running...
ERROR from element nvcuvidmpeg2dec0: No valid frames decoded before end of stream
Error: No valid frames decoded before end of stream
Returned, stopping playback
Yolo Plugin Perf Summary
Batch Size : 1
PreProcess : -nan ms Inference : -nan ms PostProcess : -nan ms Total : -nan ms per Image
Deleting pipeline
Total processing time : 0End Time now : Tue Jul 9 22:31:48 2019
But if I decode the raw mp4 file into the same size 2560x1536 h264 file.It’s ok to work.
And here is some original values that I think it’s probably useful to do such job in my code.
#define MAX_DISPLAY_LEN 64
/* The muxer output resolution must be set if the input streams will be of
* different resolution. The muxer will scale all the input frames to this
* resolution. */
#define MUXER_OUTPUT_WIDTH 1920
#define MUXER_OUTPUT_HEIGHT 1080
/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set
* based on the fastest source's framerate. */
#define MUXER_BATCH_TIMEOUT_USEC 4000000
#define TILED_OUTPUT_WIDTH 1920
#define TILED_OUTPUT_HEIGHT 1080
/* NVIDIA Decoder source pad memory feature. This feature signifies that source
* pads having this capability will push GstBuffers containing cuda buffers. */
#define GST_CAPS_FEATURES_NVMM "memory:NVMM"
/* As defined in the yolo plugins header*/
#define YOLO_UNIQUE_ID 15
Plus,one interesting thing is when I use ffmpeg to encode my raw big mp4 file into another mp4 file which is the completely same size and the same fps as the raw one.It works.Here is my command using ffmpeg to do above thing:
ffmpeg -i test.mp4 -s 2560x1536 -vcodec libx265 -r 50 another_test.mp4
Do I miss some parameters that I didn’t reset so as to match such big file?Or do you know how to avoid such bug??Thankssss
Can you please explain further about what you mean when you say - “when I directly input the raw mp4 file without any compression,which is 2560x1536, 50 fps and 512M,it seemed fail to process again.”
Are you sure the stream is valid ? Can you check if it works through gst-launch using a simple pipeline : source->decode->sink
Oh~We found out the reason was that the special video encode style for our video data was not supported by your video sdk using in deepstream.and the same issue was also met in opencv(both python and c++ version).And we pre-decode&encode using ffmpeg to solve this problem.And I have another problem now.When I use this code below modified from your demo and it works fine processing mp4 input file in ubuntu-16 desktop version system(cuda10, video sdk 9, deepstream sdk3).But when I migrate the same code into another system(server version, ubuntu-18, cuda10, video sdk9, deepstream sdk3),I got such error, on the bottom of my code.Here they are,it’s a little bit long.I bold the core code line inside.
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <iostream>
#include <glib.h>
#include <gst/gst.h>
#include "gstnvdsmeta.h"
#include <sys/time.h>
#include <ctime>
#include <iostream>
#define MAX_DISPLAY_LEN 64
#define PGIE_CLASS_ID_VEHICLE 0
#define PGIE_CLASS_ID_PERSON 2
/* The muxer output resolution must be set if the input streams will be of
* different resolution. The muxer will scale all the input frames to this
* resolution. */
#define MUXER_OUTPUT_WIDTH 2560
#define MUXER_OUTPUT_HEIGHT 1536
/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set
* based on the fastest source's framerate. */
#define MUXER_BATCH_TIMEOUT_USEC 4000000
#define TILED_OUTPUT_WIDTH 2560
#define TILED_OUTPUT_HEIGHT 1536
/* NVIDIA Decoder source pad memory feature. This feature signifies that source
* pads having this capability will push GstBuffers containing cuda buffers. */
#define GST_CAPS_FEATURES_NVMM "memory:NVMM"
/* As defined in the yolo plugins header*/
#define YOLO_UNIQUE_ID 15
gint frame_number = 0;
double frame_start;
double frame_end, fps;
/* osd_sink_pad_buffer_probe will extract metadata received on OSD sink pad
* and get a count of objects of interest */
gchar *pgie_config = NULL;
gchar *input_mp4 = NULL;
gchar *output_mp4 = NULL;
gchar *output_kitti = "/home/lijinlong/code/Yolo/deepstream_reference_apps-master/yolo/apps/deepstream-yolo/build/detected/";
gchar *video_name = NULL;
static gboolean
bus_call (GstBus * bus, GstMessage * msg, gpointer data)
{
GMainLoop *loop = (GMainLoop *) data;
switch (GST_MESSAGE_TYPE (msg)) {
case GST_MESSAGE_EOS:
g_print ("End of stream\n");
g_main_loop_quit (loop);
break;
case GST_MESSAGE_ERROR:
{
gchar *debug;
GError *error;
gst_message_parse_error (msg, &error, &debug);
g_printerr ("ERROR from element %s: %s\n", GST_OBJECT_NAME (msg->src),
error->message);
g_free (debug);
g_printerr ("Error: %s\n", error->message);
g_error_free (error);
g_main_loop_quit (loop);
break;
}
default:
break;
}
return TRUE;
}
// new ADD begin
static void
cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data)
{
GstCaps *caps = gst_pad_query_caps (decoder_src_pad, NULL);
const GstStructure *str = gst_caps_get_structure (caps, 0);
const gchar *name = gst_structure_get_name (str);
GstElement *source_bin = (GstElement *) data;
GstCapsFeatures *features = gst_caps_get_features (caps, 0);
/* Need to check if the pad created by the decodebin is for video and not
* audio. */
if (!strncmp (name, "video", 5)) {
/* Link the decodebin pad only if decodebin has picked nvidia
* decoder plugin nvdec_*. We do this by checking if the pad caps contain
* NVMM memory features. */
if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) {
/* Get the source bin ghost pad */
GstPad *bin_ghost_pad = gst_element_get_static_pad (source_bin, "src");
if (!gst_ghost_pad_set_target (GST_GHOST_PAD (bin_ghost_pad),
decoder_src_pad)) {
g_printerr ("Failed to link decoder src pad to source bin ghost pad\n");
}
gst_object_unref (bin_ghost_pad);
} else {
g_printerr ("Error: Decodebin did not pick nvidia decoder plugin.\n");
}
}
}
static GstElement *
create_source_bin (guint index, gchar * uri)
{
GstElement *bin = NULL, *uri_decode_bin = NULL;
gchar bin_name[16] = { };
g_snprintf (bin_name, 15, "source-bin-%02d", index);
/* Create a source GstBin to abstract this bin's content from the rest of the
* pipeline */
bin = gst_bin_new (bin_name);
/* Source element for reading from the uri.
* We will use decodebin and let it figure out the container format of the
* stream and the codec and plug the appropriate demux and decode plugins. */
uri_decode_bin = gst_element_factory_make ("uridecodebin", "uri-decode-bin");
if (!bin || !uri_decode_bin) {
g_printerr ("One element in source bin could not be created.\n");
return NULL;
}
/* We set the input uri to the source element */
g_object_set (G_OBJECT (uri_decode_bin), "uri", uri, NULL);
/* Connect to the "pad-added" signal of the decodebin which generates a
* callback once a new pad for raw data has beed created by the decodebin */
g_signal_connect (G_OBJECT (uri_decode_bin), "pad-added",
G_CALLBACK (cb_newpad), bin);
gst_bin_add (GST_BIN (bin), uri_decode_bin);
/* We need to create a ghost pad for the source bin which will act as a proxy
* for the video decoder src pad. The ghost pad will not have a target right
* now. Once the decode bin creates the video decoder and generates the
* cb_newpad callback, we will set the ghost pad target to the video decoder
* src pad. */
if (!gst_element_add_pad (bin, gst_ghost_pad_new_no_target ("src",
GST_PAD_SRC))) {
g_printerr ("Failed to add ghost pad in source bin\n");
return NULL;
}
return bin;
}
// new ADD end
// 计算时间
double what_time_is_it_now()
{
struct timeval time;
if (gettimeofday(&time,NULL)){
return 0;
}
return (double)time.tv_sec + (double)time.tv_usec * .000001;
}
static GstPadProbeReturn
osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info,
gpointer u_data)
{
GstMeta *gst_meta = NULL;
NvDsMeta *nvdsmeta = NULL;
gpointer state = NULL;
static GQuark _nvdsmeta_quark = 0;
GstBuffer *buf = (GstBuffer *) info->data;
NvDsFrameMeta *frame_meta = NULL;
NvOSD_RectParams *rect_params = NULL;
NvOSD_TextParams *text_params = NULL;
guint num_rects = 0, rect_index = 0;
NvDsObjectParams *obj_meta = NULL;
guint car_count = 0;
guint person_count = 0;
guint bicycle_count = 0;
guint truck_count = 0;
guint pottedplant_count = 0;
FILE *bbox_params_dump_file = NULL;
gchar bbox_file[1024] = {0};
frame_start = what_time_is_it_now();
// time_t t;
// t = time(NULL);
// std::cout << "Time now : " << ctime(&t) << std::endl;
if (!_nvdsmeta_quark)
_nvdsmeta_quark = g_quark_from_static_string (NVDS_META_STRING);
while ((gst_meta = gst_buffer_iterate_meta (buf, &state))) {
if (gst_meta_api_type_has_tag (gst_meta->info->api, _nvdsmeta_quark)) {
nvdsmeta = (NvDsMeta *) gst_meta;
/* We are interested only in intercepting Meta of type
* "NVDS_META_FRAME_INFO" as they are from our infer elements. */
if (nvdsmeta->meta_type == NVDS_META_FRAME_INFO) {
frame_meta = (NvDsFrameMeta *) nvdsmeta->meta_data;
if (frame_meta == NULL) {
g_print ("NvDS Meta contained NULL meta \n");
frame_number++;
return GST_PAD_PROBE_OK;
}
if(output_kitti)
{
g_snprintf(bbox_file, sizeof(bbox_file) - 1, "%s/%06d.txt",
output_kitti, frame_number);
bbox_params_dump_file = fopen(bbox_file, "w");
}
num_rects = frame_meta->num_rects;
/* This means we have num_rects in frame_meta->obj_params.
* Now lets iterate through them and count the number of cars,
* trucks, persons and bicycles in each frame */
for (rect_index = 0; rect_index < num_rects; rect_index++) {
obj_meta = (NvDsObjectParams *) & frame_meta->obj_params[rect_index];
rect_params = &(obj_meta->rect_params);
text_params = &(obj_meta->text_params);
if (!g_strcmp0 (obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label,
"car"))
{
if(bbox_params_dump_file)
{
int left = (int)(rect_params->left);
int top = (int)(rect_params->top);
int right = left + (int)(rect_params->width);
int bottom = top + (int)(rect_params->height);
int class_index = obj_meta->class_id;
char *text = (char*)obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label;
float prob = 0;
fprintf(bbox_params_dump_file,
"%s %d.00 %d.00 %d.00 %d.00 \n",
text, left, top, right, bottom);
std::cout << "deepstram app print" << std::endl;
std::cout << text << " " << prob << " left=" << left <<
" top=" << top << " right=" << right << " bottom=" << bottom <<
std::endl;
}
car_count++;
}
else if (!g_strcmp0 (obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label,
"person"))
{
if(bbox_params_dump_file)
{
int left = (int)(rect_params->left);
int top = (int)(rect_params->top);
int right = left + (int)(rect_params->width);
int bottom = top + (int)(rect_params->height);
int class_index = obj_meta->class_id;
char *text = (char*)obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label;
float prob = 0;
fprintf(bbox_params_dump_file,
"%s %d.00 %d.00 %d.00 %d.00 \n",
text, left, top, right, bottom);
std::cout << "deepstram app print" << std::endl;
std::cout << text << " " << prob << " left=" << left <<
" top=" << top << " right=" << right << " bottom=" << bottom <<
std::endl;
}
person_count++;
}
else if (!g_strcmp0 (obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label,
"bicycle"))
{
if(bbox_params_dump_file)
{
int left = (int)(rect_params->left);
int top = (int)(rect_params->top);
int right = left + (int)(rect_params->width);
int bottom = top + (int)(rect_params->height);
int class_index = obj_meta->class_id;
char *text = (char*)obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label;
float prob = 0;
fprintf(bbox_params_dump_file,
"%s %d.00 %d.00 %d.00 %d.00 \n",
text, left, top, right, bottom);
std::cout << "deepstram app print" << std::endl;
std::cout << text << " " << prob << " left=" << left <<
" top=" << top << " right=" << right << " bottom=" << bottom <<
std::endl;
}
bicycle_count++;
}
else if (!g_strcmp0 (obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label,
"truck"))
{
if(bbox_params_dump_file)
{
int left = (int)(rect_params->left);
int top = (int)(rect_params->top);
int right = left + (int)(rect_params->width);
int bottom = top + (int)(rect_params->height);
int class_index = obj_meta->class_id;
char *text = (char*)obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label;
float prob = 0;
fprintf(bbox_params_dump_file,
"%s %d.00 %d.00 %d.00 %d.00 \n",
text, left, top, right, bottom);
std::cout << "deepstram app print" << std::endl;
std::cout << text << " " << prob << " left=" << left <<
" top=" << top << " right=" << right << " bottom=" << bottom <<
std::endl;
}
truck_count++;
}
else if (!g_strcmp0 (obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label,
"pottedplant"))
{
if(bbox_params_dump_file)
{
int left = (int)(rect_params->left);
int top = (int)(rect_params->top);
int right = left + (int)(rect_params->width);
int bottom = top + (int)(rect_params->height);
int class_index = obj_meta->class_id;
char *text = (char*)obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label;
float prob = 0;
fprintf(bbox_params_dump_file,
"%s %d.00 %d.00 %d.00 %d.00 \n",
text, left, top, right, bottom);
std::cout << "deepstram app print" << std::endl;
std::cout << text << " " << prob << " left=" << left <<
" top=" << top << " right=" << right << " bottom=" << bottom <<
std::endl;
}
pottedplant_count++;
}
}
if(bbox_params_dump_file)
{
fclose(bbox_params_dump_file);
bbox_params_dump_file = NULL;
}
}
}
}
g_print ("Frame Number = %d Number of objects = %d "
"Car Count = %d Person Count = %d "
"Bicycle Count = %d Truck Count = %d Pottedplant Count = %d\n",
frame_number, num_rects, car_count, person_count, bicycle_count,
truck_count, pottedplant_count);
frame_number++;
frame_end = what_time_is_it_now();
fps = 1.0 / (frame_end - frame_start);
// g_print("start=%.2f\t end=%.2f\t", frame_start, frame_end);
// g_print("processing one frame time : %.2f\t", (frame_end - frame_start));
// g_print("fps = %.2f\n", fps);
return GST_PAD_PROBE_OK;
}
int
main (int argc, char *argv[])
{
GMainLoop *loop = NULL;
GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, *decoder =
NULL, *sink = NULL, *nvvidconv = NULL, *nvosd = NULL, *filter1 =
NULL, *filter2 = NULL, *yolo = NULL;
GstBus *bus = NULL;
guint bus_watch_id;
GstCaps *caps1 = NULL, *caps2 = NULL;
gulong osd_probe_id = 0;
GstPad *osd_sink_pad = NULL;
// ADD begin
GstElement *nvvidconv1 = NULL,
*filter3 = NULL,
*videoconvert = NULL,
*filter4 = NULL,
*x264enc = NULL,
*qtmux = NULL;
GstCaps *caps3 = NULL, *caps4 = NULL;
// ADD end
// new ADD begin
GstElement *streammux = NULL;
// new ADD end
/* Check input arguments */
double start_time = what_time_is_it_now();
time_t t;
t = time(NULL);
std::cout << "Start Time now : " << ctime(&t) << std::endl;
/* Standard GStreamer initialization */
gst_init (&argc, &argv);
loop = g_main_loop_new (NULL, FALSE);
/* Create gstreamer elements */
/* Create Pipeline element that will form a connection of other elements */
pipeline = gst_pipeline_new ("ds-yolo-pipeline");
[b]// new ADD begin
streammux = gst_element_factory_make ("nvstreammux", "stream-muxer");
[b]if (!pipeline || !streammux) {
g_printerr ("One element could not be created +++. pipline or streammux Exiting.\n");
return -1;
}[/b]
gst_bin_add (GST_BIN (pipeline), streammux);
GstPad *sinkpad, *srcpad;
gchar pad_name[16] = { };
GstElement *source_bin = create_source_bin (0, "file:///home/lijinlong/codebase/tensorflow_project/TensorRT_Acclerator/deepstream_reference_apps-master/yolo/apps/deepstream-yolo/build/sample_720p.mp4");
if (!source_bin) {
g_printerr ("Failed to create source bin. Exiting.\n");
return -1;
}
gst_bin_add (GST_BIN (pipeline), source_bin);
g_snprintf (pad_name, 15, "sink_%u", 0);
sinkpad = gst_element_get_request_pad (streammux, pad_name);
if (!sinkpad) {
g_printerr ("Streammux request sink pad failed. Exiting.\n");
return -1;
}
srcpad = gst_element_get_static_pad (source_bin, "src");
if (!srcpad) {
g_printerr ("Failed to get src pad of source bin. Exiting.\n");
return -1;
}
if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) {
g_printerr ("Failed to link source bin to stream muxer. Exiting.\n");
return -1;
}
[/b]
gst_object_unref (srcpad);
gst_object_unref (sinkpad);
/* Source element for reading from the file */
// source = gst_element_factory_make ("filesrc", "file-source");
/* Since the data format in the input file is elementary h264 stream,
* we need a h264parser */
/* Use convertor to convert from NV12 to RGBA as required by nvosd and yolo plugins */
nvvidconv = gst_element_factory_make ("nvvidconv", "nvvideo-converter");
/* Use yolo to run inference instead of pgie */
yolo = gst_element_factory_make ("nvyolo", "yolo-inference-engine");
/* Create OSD to draw on the converted RGBA buffer */
nvosd = gst_element_factory_make ("nvosd", "nv-onscreendisplay");
// ADD begin
/* Finally render the osd output */
nvvidconv1 = gst_element_factory_make ("nvvidconv", "nvvideo-converter1");
videoconvert = gst_element_factory_make ("videoconvert", "converter");
x264enc = gst_element_factory_make ("x264enc", "h264 encoder");
qtmux = gst_element_factory_make ("qtmux", "muxer");
// ADD end
// new ADD begin
g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height",
MUXER_OUTPUT_HEIGHT, "batch-size", 1,
"batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL);
// new ADD end
/* Finally render the osd output */
// sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer");
sink = gst_element_factory_make ("filesink", "filesink");
/* caps filter for nvvidconv to convert NV12 to RGBA as nvosd expects input
* in RGBA format */
filter1 = gst_element_factory_make ("capsfilter", "filter1");
filter2 = gst_element_factory_make ("capsfilter", "filter2");
filter3 = gst_element_factory_make ("capsfilter", "filter3");
filter4 = gst_element_factory_make ("capsfilter", "filter4");
if (!pipeline || !filter1 || !nvvidconv || !streammux || !filter2 || !nvosd || !sink || !yolo) {
g_printerr ("One element could not be created.--- Exiting.\n");
return -1;
}
/* we set the input filename to the source element */
// g_object_set (G_OBJECT (source), "location", argv[2], NULL);
g_object_set(G_OBJECT(sink), "location", "fuckshit.mp4", NULL);
g_object_set (G_OBJECT (yolo), "config-file-path", "../../../config/yolov3.txt", NULL);
/* we set the osd properties here */
g_object_set (G_OBJECT (nvosd), "font-size", 15, NULL);
/* we add a message handler */
bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
gst_object_unref (bus);
/* Set up the pipeline */
/* we add all elements into the pipeline */
// gst_bin_add_many (GST_BIN (pipeline), source, h264parser, decoder, filter1,
// nvvidconv, filter2, yolo, nvosd, sink, NULL);
gst_bin_add_many (GST_BIN (pipeline), filter1,
nvvidconv, filter2, yolo, nvosd, nvvidconv1, sink, filter3, videoconvert,
filter4, x264enc, qtmux, NULL);
caps1 = gst_caps_from_string ("video/x-raw(memory:NVMM), format=NV12");
g_object_set (G_OBJECT (filter1), "caps", caps1, NULL);
gst_caps_unref (caps1);
caps2 = gst_caps_from_string ("video/x-raw(memory:NVMM), format=RGBA");
g_object_set (G_OBJECT (filter2), "caps", caps2, NULL);
gst_caps_unref (caps2);
// ADD begin
caps3 = gst_caps_from_string ("video/x-raw, format=RGBA");
g_object_set (G_OBJECT (filter3), "caps", caps3, NULL);
gst_caps_unref (caps3);
caps4 = gst_caps_from_string ("video/x-raw, format=NV12");
g_object_set (G_OBJECT (filter4), "caps", caps4, NULL);
gst_caps_unref (caps4);
// ADD end
/* we link the elements together */
/* file-source -> h264-parser -> nvh264-decoder ->
* filter1 -> nvvidconv -> filter2 -> yolo -> nvosd -> video-renderer */
// gst_element_link_many (source, h264parser, decoder, filter1, nvvidconv,
// filter2, yolo, nvosd, sink, NULL);
gst_element_link_many (streammux, filter1, nvvidconv, filter2, yolo,
nvosd, nvvidconv1, filter3, videoconvert, filter4, x264enc, qtmux, sink, NULL);
/* Lets add probe to get informed of the meta data generated, we add probe to
* the sink pad of the osd element, since by that time, the buffer would have
* had got all the metadata. */
osd_sink_pad = gst_element_get_static_pad (nvosd, "sink");
if (!osd_sink_pad)
g_print ("Unable to get sink pad\n");
else
osd_probe_id = gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER,
osd_sink_pad_buffer_probe, NULL, NULL);
/* Set the pipeline to "playing" state */
g_print ("Now playing: %s\n", argv[2]);
gst_element_set_state (pipeline, GST_STATE_PLAYING);
/* Wait till pipeline encounters an error or EOS */
g_print ("Running...\n");
g_main_loop_run (loop);
/* Out of the main loop, clean up nicely */
g_print ("Returned, stopping playback\n");
gst_element_set_state (pipeline, GST_STATE_NULL);
g_print ("Deleting pipeline\n");
gst_object_unref (GST_OBJECT (pipeline));
g_source_remove (bus_watch_id);
g_main_loop_unref (loop);
double end_time = what_time_is_it_now();
g_print("\nTotal processing time : %d", (end_time - start_time));
time_t t2;
t2 = time(NULL);
std::cout << "End Time now : " << ctime(&t2) << std::endl;
return 0;
}
One element could not be created +++. pipline or streammux Exiting.
It seems the streammux is not created successfully in this server system which cann’t display any video or image.Do you have any ideas how to fix this or how can I use deepstream in server system?Very thanks
You can try clearing the gstreamer cache using “rm ~/.cache/gstreamer-1.0/registry.x86_64.bin” and then run “gst-inspect-1.0 -b” to see if any plugins are getting blacklisted.
That being said, DS 3.0 is not officially supported on ubuntu 18.04, so please wait for DS4.0(to be released soon) which is supported on Ubuntu 18.04.
Hi~I tried your method.And I got this output:
Blacklisted files:
Total count: 0 blacklisted files
It seemed nothing got sucked.But I just can’t run the same code in this machine(ubuntu18 server system).
And will DS4 support linux sever system,such as ubuntu18 server?And I am quiet looking forward to DS4.
Plus could you provide more documents and more informations in your offical github project while releasing DS4.
Looking forward…
Deepstream 4.0 has been released and Ubuntu 18.04 is supported - https://developer.nvidia.com/deepstream-sdk