Hi all,
I’ve been trying to use DeepStream in ROS on Jetson Nano. It’s a simple ROS Nodelet that will receive images as ROS sensor_msgs/Image and feeds them to the DeepStream pipeline.
Here’s my DeepStream pipeline code:
//GStreamer
gst_init(0,0);
loop = g_main_loop_new (NULL, FALSE);
loop_thread_ = std::thread(
[&]() {
NODELET_DEBUG("GMainLoop started.");
g_main_loop_run(loop);
g_main_loop_unref(loop);
loop = nullptr;
NODELET_INFO("GMainLoop terminated.");
});
pthread_setname_np(loop_thread_.native_handle(), "g_main_loop_run");
guint bus_watch_id = 0;
pipeline = gst_pipeline_new ("deepstream-pipeline");
appsrc = gst_element_factory_make("appsrc", "source");
queue = gst_element_factory_make ("queue", "srcqueue");
gst_app_src_set_stream_type(GST_APP_SRC_CAST(appsrc), GST_APP_STREAM_TYPE_STREAM);
gst_app_src_set_latency(GST_APP_SRC_CAST(appsrc), 0, -1);
g_object_set(GST_OBJECT(appsrc),
"format", GST_FORMAT_TIME,
"is-live", true,
"max-bytes", 0,
"do-timestamp", true,
NULL);
vidconv = gst_element_factory_make ("nvvideoconvert", "vidconv");
filter = gst_element_factory_make ("capsfilter", "capsfilter");
GstCaps *vid_caps = gst_caps_from_string ("video/x-raw(memory:NVMM), format=I420");
g_object_set(G_OBJECT(filter), "caps", vid_caps, NULL);
gst_caps_unref( vid_caps);
streammux = gst_element_factory_make ("nvstreammux", "stream-muxer");
if (!pipeline || !streammux)
ROS_FATAL("One element could not be created. Exiting!");
else
ROS_INFO("done pipeline and streammux");
pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine");
nvtracker = gst_element_factory_make ("nvtracker", "tracker");
nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter");
nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay");
transform = gst_element_factory_make ("nvegltransform", "nvegl-transform");
sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer");
if (!appsrc || !pgie || !nvtracker || !nvvidconv || !nvosd || !transform || !sink)
ROS_FATAL("One element could not be created. Exiting!");
else
ROS_INFO("done pgie...");
g_object_set (G_OBJECT (streammux), "width", 672, "height",
376, "batch-size", 1,
"batched-push-timeout", 6000000, NULL);
g_object_set (G_OBJECT (pgie), "config-file-path", pgie_config_file.c_str(), NULL);
if (!set_tracker_properties(nvtracker, tracker_config_file))
ROS_FATAL("Failed to set tracker properties. Exiting!");
else
ROS_INFO("done tracker");
bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
bus_watch_id = gst_bus_add_watch (bus, AISHumanNodelet::bus_message_cb_wrapper, loop);
gst_object_unref (bus);
gst_bin_add_many (GST_BIN (pipeline),
appsrc, queue, filter, vidconv, streammux, pgie, nvtracker,
nvvidconv, nvosd, transform, sink, NULL);
GstPad *sinkpad, *srcpad;
gchar pad_name_sink[16] = "sink_0";
gchar pad_name_src[16] = "src";
sinkpad = gst_element_get_request_pad (streammux, pad_name_sink);
if (!sinkpad)
ROS_FATAL("Streammux request sink pad failed. Exiting!");
srcpad = gst_element_get_static_pad (vidconv, pad_name_src);
if (!srcpad)
ROS_FATAL("Decoder request src pad failed. Exiting.");
if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK)
ROS_FATAL("Failed to link converter to stream muxer. Exiting.");
gst_object_unref (sinkpad);
gst_object_unref (srcpad);
if (!gst_element_link_many ( appsrc, queue, filter, vidconv, NULL))
ROS_FATAL("source and converter Elements could not be linked. Exiting!");
if (!gst_element_link_many ( streammux, pgie, nvtracker,
nvvidconv, nvosd, transform, sink, NULL))
ROS_FATAL("Elements could not be linked. Exiting!");
else
ROS_INFO("done link streamux pgie tracker ...");
osd_sink_pad = gst_element_get_static_pad (nvosd, "sink");
if (!osd_sink_pad)
ROS_ERROR ("Unable to get sink pad");
else
gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER,
osd_sink_pad_buffer_probe, NULL, NULL);
ROS_INFO("inited");
gst_element_set_state(pipeline, GST_STATE_PAUSED);
if (gst_element_get_state(pipeline, nullptr, nullptr, 1_sec) == GST_STATE_CHANGE_FAILURE) {
NODELET_ERROR("GST: state change error. Check your pipeline.");
}
else {
NODELET_INFO("GST: pipeline paused.");
}
When I receive the first image, read the frame information and apply the following caps to the appsrc, (the image is from a ZED camera that’s where the weird resolution come from):
GST_EVENT gstevent.c:814:gst_event_new_caps: creating caps event video/x-raw, format=(string)BGRA, width=(int)672, height=(int)376, framerate=(fraction)0/1, pixel-aspect-ratio=(fraction)1/1, interlace-mode=(string)progressive
But the pipeline breaks as soon as the first image is fed to it:
0:00:04.726038213 32701 0x5581d08e80 INFO GST_EVENT gstevent.c:814:gst_event_new_caps: creating caps event video/x-raw, format=(string)BGRA, width=(int)672, height=(int)376, framerate=(fraction)0/1, pixel-aspect-ratio=(fraction)1/1, interlace-mode=(string)progressive
0:00:04.727931405 32701 0x5581d08e80 INFO GST_EVENT gstevent.c:895:gst_event_new_segment: creating segment event time segment start=0:00:00.000000000, offset=0:00:00.000000000, stop=99:99:99.999999999, rate=1.000000, applied_rate=1.000000, flags=0x00, time=0:00:00.000000000, base=0:00:00.000000000, position 0:00:00.000000000, duration 99:99:99.999999999
0:00:04.728064944 32701 0x5581d08e80 INFO basesrc gstbasesrc.c:2945:gst_base_src_loop:<source> marking pending DISCONT
0:00:04.728427645 32701 0x5581d08e80 WARN basesrc gstbasesrc.c:3055:gst_base_src_loop:<source> error: Internal data stream error.
0:00:04.728461811 32701 0x5581d08e80 WARN basesrc gstbasesrc.c:3055:gst_base_src_loop:<source> error: streaming stopped, reason not-negotiated (-4)
0:00:04.728641651 32701 0x5581d08e80 INFO GST_ERROR_SYSTEM gstelement.c:2145:gst_element_message_full_with_details:<source> posting message: Internal data stream error.
0:00:04.728796595 32701 0x5581d08e80 INFO GST_ERROR_SYSTEM gstelement.c:2172:gst_element_message_full_with_details:<source> posted error message: Internal data stream error.
0:00:04.728941436 32701 0x5581d08e80 INFO task gsttask.c:316:gst_task_func:<source:src> Task going to paused
0:00:04.729076902 32701 0x5581d08c50 INFO task gsttask.c:316:gst_task_func:<srcqueue:src> Task going to paused
[ERROR] [1573107885.088169037]: GST: bus: Internal data stream error.
[ERROR] [1573107885.088285598]: GST: debug: gstbasesrc.c(3055): gst_base_src_loop (): /GstPipeline:deepstream-pipeline/GstAppSrc:source:
streaming stopped, reason not-negotiated (-4)
I have tried adding different decoders/converters/caps filters between the appsrc and nvstreammux with no luck.
I’d appreciate any help, this will be an OpenSource contribution to the ROS community.