Deepstream in ROS

Hi all,

I’ve been trying to use DeepStream in ROS on Jetson Nano. It’s a simple ROS Nodelet that will receive images as ROS sensor_msgs/Image and feeds them to the DeepStream pipeline.

Here’s my DeepStream pipeline code:

//GStreamer
      gst_init(0,0);
      loop = g_main_loop_new (NULL, FALSE);
      loop_thread_ = std::thread(
                [&]() {
                        NODELET_DEBUG("GMainLoop started.");
                        g_main_loop_run(loop);
                        g_main_loop_unref(loop);
                        loop = nullptr;
                        NODELET_INFO("GMainLoop terminated.");
                });
      pthread_setname_np(loop_thread_.native_handle(), "g_main_loop_run");
      guint bus_watch_id = 0;
      pipeline = gst_pipeline_new ("deepstream-pipeline");
      appsrc = gst_element_factory_make("appsrc", "source");
      queue = gst_element_factory_make ("queue", "srcqueue");
      gst_app_src_set_stream_type(GST_APP_SRC_CAST(appsrc), GST_APP_STREAM_TYPE_STREAM);
      gst_app_src_set_latency(GST_APP_SRC_CAST(appsrc), 0, -1);
      g_object_set(GST_OBJECT(appsrc),
                        "format", GST_FORMAT_TIME,
                        "is-live", true,
                        "max-bytes", 0,
                        "do-timestamp", true,
                        NULL);
      vidconv = gst_element_factory_make ("nvvideoconvert", "vidconv");
      filter = gst_element_factory_make ("capsfilter", "capsfilter");
      GstCaps *vid_caps = gst_caps_from_string ("video/x-raw(memory:NVMM), format=I420");
      g_object_set(G_OBJECT(filter), "caps", vid_caps, NULL);
      gst_caps_unref( vid_caps);
      streammux = gst_element_factory_make ("nvstreammux", "stream-muxer");
      if (!pipeline || !streammux)
         ROS_FATAL("One element could not be created. Exiting!");
      else
         ROS_INFO("done pipeline and streammux");
      pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine");
      nvtracker = gst_element_factory_make ("nvtracker", "tracker");
      nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter");
      nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay");
      transform = gst_element_factory_make ("nvegltransform", "nvegl-transform");
      sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer");

      if (!appsrc || !pgie || !nvtracker || !nvvidconv || !nvosd || !transform || !sink)
         ROS_FATAL("One element could not be created. Exiting!");
      else
         ROS_INFO("done pgie...");

      g_object_set (G_OBJECT (streammux), "width", 672, "height",
          376, "batch-size", 1,
          "batched-push-timeout", 6000000, NULL);
      g_object_set (G_OBJECT (pgie), "config-file-path", pgie_config_file.c_str(), NULL);
      if (!set_tracker_properties(nvtracker, tracker_config_file))
        ROS_FATAL("Failed to set tracker properties. Exiting!");
      else
         ROS_INFO("done tracker");
      bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
      bus_watch_id = gst_bus_add_watch (bus, AISHumanNodelet::bus_message_cb_wrapper, loop);
      gst_object_unref (bus);
      gst_bin_add_many (GST_BIN (pipeline),
          appsrc, queue, filter, vidconv, streammux, pgie, nvtracker,
          nvvidconv, nvosd, transform, sink, NULL);

      GstPad *sinkpad, *srcpad;
      gchar pad_name_sink[16] = "sink_0";
      gchar pad_name_src[16] = "src";

      sinkpad = gst_element_get_request_pad (streammux, pad_name_sink);
      if (!sinkpad)
            ROS_FATAL("Streammux request sink pad failed. Exiting!");

      srcpad = gst_element_get_static_pad (vidconv, pad_name_src);
      if (!srcpad)
            ROS_FATAL("Decoder request src pad failed. Exiting.");

      if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK)
            ROS_FATAL("Failed to link converter to stream muxer. Exiting.");

      gst_object_unref (sinkpad);
      gst_object_unref (srcpad);

      if (!gst_element_link_many ( appsrc, queue, filter, vidconv, NULL))
        ROS_FATAL("source and converter Elements could not be linked. Exiting!");
      if (!gst_element_link_many ( streammux, pgie, nvtracker,
          nvvidconv, nvosd, transform, sink, NULL))
        ROS_FATAL("Elements could not be linked. Exiting!");
      else
        ROS_INFO("done link streamux pgie tracker ...");

      osd_sink_pad = gst_element_get_static_pad (nvosd, "sink");
      if (!osd_sink_pad)
        ROS_ERROR ("Unable to get sink pad");
      else
        gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER,
            osd_sink_pad_buffer_probe, NULL, NULL);
      ROS_INFO("inited");

      gst_element_set_state(pipeline, GST_STATE_PAUSED);
        if (gst_element_get_state(pipeline, nullptr, nullptr, 1_sec) == GST_STATE_CHANGE_FAILURE) {
                NODELET_ERROR("GST: state change error. Check your pipeline.");
        }
        else {
                NODELET_INFO("GST: pipeline paused.");
        }

When I receive the first image, read the frame information and apply the following caps to the appsrc, (the image is from a ZED camera that’s where the weird resolution come from):

GST_EVENT gstevent.c:814:gst_event_new_caps: creating caps event video/x-raw, format=(string)BGRA, width=(int)672, height=(int)376, framerate=(fraction)0/1, pixel-aspect-ratio=(fraction)1/1, interlace-mode=(string)progressive

But the pipeline breaks as soon as the first image is fed to it:

0:00:04.726038213 32701   0x5581d08e80 INFO               GST_EVENT gstevent.c:814:gst_event_new_caps: creating caps event video/x-raw, format=(string)BGRA, width=(int)672, height=(int)376, framerate=(fraction)0/1, pixel-aspect-ratio=(fraction)1/1, interlace-mode=(string)progressive
0:00:04.727931405 32701   0x5581d08e80 INFO               GST_EVENT gstevent.c:895:gst_event_new_segment: creating segment event time segment start=0:00:00.000000000, offset=0:00:00.000000000, stop=99:99:99.999999999, rate=1.000000, applied_rate=1.000000, flags=0x00, time=0:00:00.000000000, base=0:00:00.000000000, position 0:00:00.000000000, duration 99:99:99.999999999
0:00:04.728064944 32701   0x5581d08e80 INFO                 basesrc gstbasesrc.c:2945:gst_base_src_loop:<source> marking pending DISCONT
0:00:04.728427645 32701   0x5581d08e80 WARN                 basesrc gstbasesrc.c:3055:gst_base_src_loop:<source> error: Internal data stream error.
0:00:04.728461811 32701   0x5581d08e80 WARN                 basesrc gstbasesrc.c:3055:gst_base_src_loop:<source> error: streaming stopped, reason not-negotiated (-4)
0:00:04.728641651 32701   0x5581d08e80 INFO        GST_ERROR_SYSTEM gstelement.c:2145:gst_element_message_full_with_details:<source> posting message: Internal data stream error.
0:00:04.728796595 32701   0x5581d08e80 INFO        GST_ERROR_SYSTEM gstelement.c:2172:gst_element_message_full_with_details:<source> posted error message: Internal data stream error.
0:00:04.728941436 32701   0x5581d08e80 INFO                    task gsttask.c:316:gst_task_func:<source:src> Task going to paused
0:00:04.729076902 32701   0x5581d08c50 INFO                    task gsttask.c:316:gst_task_func:<srcqueue:src> Task going to paused
[ERROR] [1573107885.088169037]: GST: bus: Internal data stream error.
[ERROR] [1573107885.088285598]: GST: debug: gstbasesrc.c(3055): gst_base_src_loop (): /GstPipeline:deepstream-pipeline/GstAppSrc:source:
streaming stopped, reason not-negotiated (-4)

I have tried adding different decoders/converters/caps filters between the appsrc and nvstreammux with no luck.

I’d appreciate any help, this will be an OpenSource contribution to the ROS community.

Ok streaming the ROS messages on UDP and feeding the UDP stream to deepstream-app itself crashed with the same error.

I tried streaming the ROS messages on UDP with h-264 encoding using the following pipeline:

videorate ! 
                video/x-raw,framerate=30/1 ! 
                videoconvert ! 
                x264enc speed-preset=ultrafast tune=zerolatency byte-stream=true threads=1 key-int-max=15 intra-refresh=true ! 
                video/x-h264, profile=baseline ! 
                rtph264pay ! 
                udpsink host=127.0.0.1 port=5000

I can see the video is streaming OK by:

gst-launch-1.0 -v udpsrc port=5000 ! application/x-rtp,clock-rate=90000,payload=96 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! autovideosink

However if I feed the stream to a working DeepSream pipeline by changing the source in the confing file as:

[source0]
type=2
<b>uri=udp://127.0.0.1:5000</b>
num-sources=1
gpu-id=0
cudadec-memtype=0

And run thee deepstream-app it crashed with the following errors:

0:00:05.664441316 16400     0x2fd84e30 INFO        GST_ERROR_SYSTEM gstelement.c:2145:gst_element_message_full_with_details:<typefind> posting message: Could not determine type of stream.
0:00:05.664618976 16400     0x2fd84e30 INFO        GST_ERROR_SYSTEM gstelement.c:2172:gst_element_message_full_with_details:<typefind> posted error message: Could not determine type of stream.
ERROR from typefind: Could not determine type of stream.
Debug info: gsttypefindelement.c(1007): gst_type_find_element_chain_do_typefinding (): /GstPipeline:pipeline/GstBin:multi_src_bin/GstBin:src_sub_bin0/GstURIDecodeBin:src_elem/GstDecodeBin:decodebin0/GstTypeFindElement:typefind
0:00:05.665037839 16400     0x2fd84e30 INFO                 basesrc gstbasesrc.c:2965:gst_base_src_loop:<source> pausing after gst_pad_push() = error
0:00:05.665110497 16400     0x2fd84e30 WARN                 basesrc gstbasesrc.c:3055:gst_base_src_loop:<source> error: Internal data stream error.
0:00:05.665137008 16400     0x2fd84e30 WARN                 basesrc gstbasesrc.c:3055:gst_base_src_loop:<source> error: streaming stopped, reason error (-5)
0:00:05.665198051 16400     0x2fd84e30 INFO        GST_ERROR_SYSTEM gstelement.c:2145:gst_element_message_full_with_details:<source> posting message: Internal data stream error.
0:00:05.665278730 16400     0x2fd84e30 INFO        GST_ERROR_SYSTEM gstelement.c:2172:gst_element_message_full_with_details:<source> posted error message: Internal data stream error.
0:00:05.665340190 16400     0x2fd84e30 INFO                    task gsttask.c:316:gst_task_func:<source:src> Task going to paused
ERROR from source: Internal data stream error.
Debug info: gstbasesrc.c(3055): gst_base_src_loop (): /GstPipeline:pipeline/GstBin:multi_src_bin/GstBin:src_sub_bin0/GstURIDecodeBin:src_elem/GstUDPSrc:source:
streaming stopped, reason error (-5)
Quitting
.
.
.
App run failed

Can you try source type 4