DeepStream http streams EOS crash and reconnecting

• Hardware Platform (Jetson / GPU)
GPU
• DeepStream Version
6.2
• NVIDIA GPU Driver Version (valid for GPU only)
525.125.06
• Issue Type( questions, new requirements, bugs)
Bugs
• How to reproduce the issue ? (This is for bugs. Including which sample app is using, the configuration files content, the command line used and other details for reproducing)

I have a problem while using http stream which restream mjpeg frames. I’m currently reading two cameras from http and if I disconnect one stream I get nvstreammux: Successfully handled EOS for source_id=1 and if I disconnect another one aswell I get nvstreammux: Successfully handled EOS for source_id=0
But the problem occurs that sometimes the DeepStream app crashes, but sometimes it doesn’t and I can’t reproduce it to be consistent and not crash. I added drop-pipeline-eos=1 to streammux for DeepStream not to crash.

Logs when DeepStream doesn’t crash:

**PERF:  FPS 0 (Avg)	FPS 1 (Avg)	
**PERF:  28.82 (28.84)	25.75 (24.98)	
nvstreammux: Successfully handled EOS for source_id=1
**PERF:  28.82 (28.84)	25.18 (24.89)	
**PERF:  28.82 (28.84)	0.00 (23.80)	
nvstreammux: Successfully handled EOS for source_id=0
**PERF:  28.77 (27.97)	0.00 (22.80)	
**PERF:  0.00 (26.84)	0.00 (21.88)	
**PERF:  0.00 (25.80)	0.00 (21.03)	
**PERF:  0.00 (24.84)	0.00 (20.24)	
**PERF:  0.00 (23.94)	0.00 (19.51)	
**PERF:  0.00 (23.11)	0.00 (18.83)

Logs when it does crash:

**PERF:  FPS 0 (Avg)	FPS 1 (Avg)	
**PERF:  29.47 (29.42)	25.44 (25.39)	
nvstreammux: Successfully handled EOS for source_id=1
**PERF:  28.81 (29.03)	25.43 (14.52)	
**PERF:  28.81 (29.00)	0.00 (9.21)	
**PERF:  28.81 (28.93)	0.00 (6.75)	
**PERF:  28.82 (28.90)	0.00 (5.32)	
nvstreammux: Successfully handled EOS for source_id=0
** INFO: <bus_callback:263>: Received EOS. Exiting ...

Quitting
App run successful

Another issue I have is that http stream don’t reconnect if they start working again after EOS. I tried adding rtsp-reconnect-attempts=-1 and rtsp-reconnect-interval-sec=60 to source but it didn’t help. How could I make DeepStream reconnect to http streams?

My full config file:

[application]
enable-perf-measurement=1
perf-measurement-interval-sec=15

[tiled-display]
enable=0
rows=1
columns=1
width=1016
height=504
gpu-id=0
nvbuf-memory-type=0

[source0]
enable=1
type=2
uri=http://192.168.0.75:8000/camera/mjpeg
num-sources=1
gpu-id=0
cudadec-memtype=0
rtsp-reconnect-attempts=-1
rtsp-reconnect-interval-sec=60

[source1]
enable=1
type=2
uri=http://192.168.0.196:8000/camera/mjpeg
num-sources=1
gpu-id=0
cudadec-memtype=0
rtsp-reconnect-attempts=-1
rtsp-reconnect-interval-sec=60

[sink0]
enable=1
type=1
sync=0
source-id=0
gpu-id=0
nvbuf-memory-type=0

[sink1]
enable=1
type=1
sync=0
source-id=1
gpu-id=0
nvbuf-memory-type=0

[osd]
enable=0
gpu-id=0
border-width=2
text-size=20
text-color=1;1;1;1;
text-bg-color=0.3;0.3;0.3;1
font=Serif
show-clock=0
clock-x-offset=800
clock-y-offset=820
clock-text-size=12
clock-color=1;0;0;0
nvbuf-memory-type=0

[streammux]
gpu-id=0
live-source=1
batch-size=2
batched-push-timeout=40000
width=1920
height=1080
enable-padding=0
nvbuf-memory-type=0
drop-pipeline-eos=1

[pre-process]
enable=1
config-file=config_preprocess.txt

[primary-gie]
enable=1
gpu-id=0
batch-size=2
gie-unique-id=1
process-mode=1
nvbuf-memory-type=0
config-file=config_infer_primary.txt

Judging from your log, there is no crash. The difference is that the latter receives the EOS message and then exits normally.

Can you use gdb --args deepstream -c your configuration file to dump the crash stack?

These two properties are used for rtsp stream reconnection, not applicable to http stream.

you can refer these two functions : watch_source_status and rtspsrc_monitor_probe_func in /opt/nvidia/deepstream/deepstream/sources/apps/apps-common/src/deepstream_source_bin.c

Thank you @junshengy for reply.
Yes, the app doesn’t crash, but it exits successfully, so my question would be how could I have it running indefinitely after it receives EOS for all streams and after EOS if stream is back online it would reconnect to it.

I have modified /opt/nvidia/deepstream/deepstream/sources/apps/apps-common/src/deepstream_source_bin.c and its create_uridecode_src_bin function, here it is:

static gboolean
create_uridecode_src_bin (NvDsSourceConfig * config, NvDsSrcBin * bin)
{
  gboolean ret = FALSE;
  GstCaps *caps = NULL;
  GstCapsFeatures *feature = NULL;
  bin->config = config;
  gchar elem_name[50];
  bin->rtsp_reconnect_interval_sec = config->rtsp_reconnect_interval_sec;
  bin->rtsp_reconnect_attempts = config->rtsp_reconnect_attempts;
  bin->num_rtsp_reconnects = 0;

  bin->src_elem = gst_element_factory_make (NVDS_ELEM_SRC_URI, "src_elem");
  if (!bin->src_elem) {
    NVGSTDS_ERR_MSG_V ("Could not create element 'src_elem'");
    goto done;
  }

  if (config->dewarper_config.enable) {
    if (!create_dewarper_bin (&config->dewarper_config, &bin->dewarper_bin)) {
      g_print ("Creating Dewarper bin failed \n");
      goto done;
    }
  }
  bin->latency = config->latency;
  bin->udp_buffer_size = config->udp_buffer_size;

  if (g_strrstr (config->uri, "file:/")) {
    config->live_source = FALSE;
  }
  if (g_strrstr (config->uri, "rtsp://") == config->uri) {
    configure_source_for_ntp_sync (bin->src_elem);
  }

  g_object_set (G_OBJECT (bin->src_elem), "uri", config->uri, NULL);
  g_signal_connect (G_OBJECT (bin->src_elem), "pad-added",
      G_CALLBACK (cb_newpad), bin);
  g_signal_connect (G_OBJECT (bin->src_elem), "child-added",
      G_CALLBACK (decodebin_child_added), bin);
  g_signal_connect (G_OBJECT (bin->src_elem), "source-setup",
      G_CALLBACK (cb_sourcesetup), bin);
  bin->cap_filter = gst_element_factory_make (NVDS_ELEM_QUEUE, "queue");
  if (!bin->cap_filter) {
    NVGSTDS_ERR_MSG_V ("Could not create 'queue'");
    goto done;
  }

  g_snprintf (elem_name, sizeof (elem_name), "dec_que%d", bin->bin_id);
  bin->dec_que = gst_element_factory_make ("queue", elem_name);
  if (!bin->dec_que) {
    NVGSTDS_ERR_MSG_V ("Failed to create '%s'", elem_name);
    goto done;
  }

  if (bin->rtsp_reconnect_interval_sec > 0) {
    NVGSTDS_ELEM_ADD_PROBE(bin->rtspsrc_monitor_probe, bin->dec_que, "sink", rtspsrc_monitor_probe_func, GST_PAD_PROBE_TYPE_BUFFER, bin);
    install_mux_eosmonitor_probe = TRUE;
  } else {
    NVGSTDS_ELEM_ADD_PROBE (bin->rtspsrc_monitor_probe, bin->dec_que,
        "sink", rtspsrc_monitor_probe_func,
        GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, bin);
  }

  bin->nvvidconv =
      gst_element_factory_make (NVDS_ELEM_VIDEO_CONV, "nvvidconv_elem");
  if (!bin->nvvidconv) {
    NVGSTDS_ERR_MSG_V ("Could not create element 'nvvidconv_elem'");
    goto done;
  }

  if (config->video_format) {
    caps = gst_caps_new_simple ("video/x-raw", "format", G_TYPE_STRING, config->video_format, NULL);
  } else {
    caps = gst_caps_new_empty_simple ("video/x-raw");
  }
  feature = gst_caps_features_new ("memory:NVMM", NULL);
  gst_caps_set_features (caps, 0, feature);

  bin->cap_filter1 =
      gst_element_factory_make (NVDS_ELEM_CAPS_FILTER,
      "src_cap_filter_nvvidconv");
  if (!bin->cap_filter1) {
    NVGSTDS_ERR_MSG_V ("Could not create 'queue'");
    goto done;
  }

  g_object_set (G_OBJECT (bin->cap_filter1), "caps", caps, NULL);
  gst_caps_unref (caps);

  g_object_set_data (G_OBJECT (bin->cap_filter), SRC_CONFIG_KEY, config);

  gst_bin_add_many (GST_BIN (bin->bin), bin->src_elem, bin->cap_filter,
      bin->nvvidconv, bin->cap_filter1, NULL);

  NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->cap_filter1, "src");

  bin->fakesink = gst_element_factory_make ("fakesink", "src_fakesink");
  if (!bin->fakesink) {
    NVGSTDS_ERR_MSG_V ("Could not create 'src_fakesink'");
    goto done;
  }

  bin->fakesink_queue = gst_element_factory_make ("queue", "fakequeue");
  if (!bin->fakesink_queue) {
    NVGSTDS_ERR_MSG_V ("Could not create 'fakequeue'");
    goto done;
  }

  bin->tee = gst_element_factory_make ("tee", NULL);
  if (!bin->tee) {
    NVGSTDS_ERR_MSG_V ("Could not create 'tee'");
    goto done;
  }
  gst_bin_add_many (GST_BIN (bin->bin), bin->fakesink, bin->tee,
      bin->fakesink_queue, NULL);


  NVGSTDS_LINK_ELEMENT (bin->fakesink_queue, bin->fakesink);

  if (config->dewarper_config.enable) {
    gst_bin_add_many (GST_BIN (bin->bin), bin->dewarper_bin.bin, NULL);
    NVGSTDS_LINK_ELEMENT (bin->tee, bin->dewarper_bin.bin);
    NVGSTDS_LINK_ELEMENT (bin->dewarper_bin.bin, bin->cap_filter);
  } else {
    link_element_to_tee_src_pad (bin->tee, bin->cap_filter);
  }
  NVGSTDS_LINK_ELEMENT (bin->cap_filter, bin->nvvidconv);
  NVGSTDS_LINK_ELEMENT (bin->nvvidconv, bin->cap_filter1);
  link_element_to_tee_src_pad (bin->tee, bin->fakesink_queue);

  g_object_set (G_OBJECT (bin->fakesink), "sync", FALSE, "async", FALSE, NULL);
  g_object_set (G_OBJECT (bin->fakesink), "enable-last-sample", FALSE, NULL);

  ret = TRUE;

  g_timeout_add (1000, watch_source_status, bin);

  GST_CAT_DEBUG (NVDS_APP,
      "Decode bin created. Waiting for a new pad from decodebin to link");

But reconnect still doesn’t work, could you point me to some direction what else I should modify so that it would work?

Thank you in advance!

You need to add the queue to the pipeline, otherwise the monitor probe will not take effect.

Thank you @junshengy now the monitor probe works, however function check_rtsp_reconnection_attempts (NvDsSrcBin * src_bin) doesn’t ever get triggered. What other modifications are needed?

Also the DeepStream app now quits, even though I have set drop-pipeline-eos=1 in Streammux which I thought should keep the pipeline running forever?

Here are DeepStream logs:

** INFO: <bus_callback:226>: Pipeline running


**PERF:  FPS 0 (Avg)
**PERF:  61.93 (55.00)
**PERF:  23.92 (25.80)
**PERF:  23.90 (24.83)
**PERF:  23.91 (24.56)
**PERF:  23.94 (24.42)
**PERF:  23.96 (20.98)
**PERF:  0.00 (17.52)
**PERF:  0.00 (15.04)
**PERF:  0.00 (13.17)
**PERF:  0.00 (11.72)
**PERF:  0.00 (10.55)
ERROR from source: Internal data stream error.
Debug info: gstbasesrc.c(3072): gst_base_src_loop (): /GstPipeline:pipeline/GstBin:multi_src_bin/GstBin:src_sub_bin0/GstURIDecodeBin:src_elem/GstSoupHTTPSrc:source:
streaming stopped, reason error (-5)
nvstreammux: Successfully handled EOS for source_id=0
Quitting
App run failed

Here is the current function:

static gboolean
create_uridecode_src_bin (NvDsSourceConfig * config, NvDsSrcBin * bin)
{
  gboolean ret = FALSE;
  GstCaps *caps = NULL;
  GstCapsFeatures *feature = NULL;
  bin->config = config;
  gchar elem_name[50];
  bin->rtsp_reconnect_interval_sec = config->rtsp_reconnect_interval_sec;
  bin->rtsp_reconnect_attempts = config->rtsp_reconnect_attempts;
  bin->num_rtsp_reconnects = 0;

  bin->src_elem = gst_element_factory_make (NVDS_ELEM_SRC_URI, "src_elem");
  if (!bin->src_elem) {
    NVGSTDS_ERR_MSG_V ("Could not create element 'src_elem'");
    goto done;
  }

  if (config->dewarper_config.enable) {
    if (!create_dewarper_bin (&config->dewarper_config, &bin->dewarper_bin)) {
      g_print ("Creating Dewarper bin failed \n");
      goto done;
    }
  }
  bin->latency = config->latency;
  bin->udp_buffer_size = config->udp_buffer_size;

  if (g_strrstr (config->uri, "file:/")) {
    config->live_source = FALSE;
  }
  if (g_strrstr (config->uri, "rtsp://") == config->uri) {
    configure_source_for_ntp_sync (bin->src_elem);
  }

  g_object_set (G_OBJECT (bin->src_elem), "uri", config->uri, NULL);
  g_object_set (G_OBJECT (bin->src_elem), "drop-on-latency", TRUE, NULL);
  g_signal_connect (G_OBJECT (bin->src_elem), "pad-added",
      G_CALLBACK (cb_newpad), bin);
  g_signal_connect (G_OBJECT (bin->src_elem), "child-added",
      G_CALLBACK (decodebin_child_added), bin);
  g_signal_connect (G_OBJECT (bin->src_elem), "source-setup",
      G_CALLBACK (cb_sourcesetup), bin);
  bin->cap_filter = gst_element_factory_make (NVDS_ELEM_QUEUE, "queue");
  if (!bin->cap_filter) {
    NVGSTDS_ERR_MSG_V ("Could not create 'queue'");
    goto done;
  }

  g_snprintf (elem_name, sizeof (elem_name), "dec_que%d", bin->bin_id);
  bin->dec_que = gst_element_factory_make ("queue", elem_name);
  if (!bin->dec_que) {
    NVGSTDS_ERR_MSG_V ("Failed to create '%s'", elem_name);
    goto done;
  }

  if (bin->rtsp_reconnect_interval_sec > 0) {
    NVGSTDS_ELEM_ADD_PROBE(bin->rtspsrc_monitor_probe, bin->dec_que, "sink", rtspsrc_monitor_probe_func, GST_PAD_PROBE_TYPE_BUFFER, bin);
    install_mux_eosmonitor_probe = TRUE;
  } else {
    NVGSTDS_ELEM_ADD_PROBE (bin->rtspsrc_monitor_probe, bin->dec_que,
        "sink", rtspsrc_monitor_probe_func,
        GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, bin);
  }

  bin->nvvidconv =
      gst_element_factory_make (NVDS_ELEM_VIDEO_CONV, "nvvidconv_elem");
  if (!bin->nvvidconv) {
    NVGSTDS_ERR_MSG_V ("Could not create element 'nvvidconv_elem'");
    goto done;
  }

  if (config->video_format) {
    caps = gst_caps_new_simple ("video/x-raw", "format", G_TYPE_STRING, config->video_format, NULL);
  } else {
    caps = gst_caps_new_empty_simple ("video/x-raw");
  }
  feature = gst_caps_features_new ("memory:NVMM", NULL);
  gst_caps_set_features (caps, 0, feature);

  bin->cap_filter1 =
      gst_element_factory_make (NVDS_ELEM_CAPS_FILTER,
      "src_cap_filter_nvvidconv");
  if (!bin->cap_filter1) {
    NVGSTDS_ERR_MSG_V ("Could not create 'queue'");
    goto done;
  }

  g_object_set (G_OBJECT (bin->cap_filter1), "caps", caps, NULL);
  gst_caps_unref (caps);

  g_object_set_data (G_OBJECT (bin->cap_filter), SRC_CONFIG_KEY, config);

  gst_bin_add_many (GST_BIN (bin->bin), bin->src_elem, bin->cap_filter,
      bin->dec_que, bin->nvvidconv, bin->cap_filter1, NULL);

  NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->cap_filter1, "src");

  bin->fakesink = gst_element_factory_make ("fakesink", "src_fakesink");
  if (!bin->fakesink) {
    NVGSTDS_ERR_MSG_V ("Could not create 'src_fakesink'");
    goto done;
  }

  bin->fakesink_queue = gst_element_factory_make ("queue", "fakequeue");
  if (!bin->fakesink_queue) {
    NVGSTDS_ERR_MSG_V ("Could not create 'fakequeue'");
    goto done;
  }

  bin->tee = gst_element_factory_make ("tee", NULL);
  if (!bin->tee) {
    NVGSTDS_ERR_MSG_V ("Could not create 'tee'");
    goto done;
  }
  gst_bin_add_many (GST_BIN (bin->bin), bin->fakesink, bin->tee,
      bin->fakesink_queue, NULL);

  NVGSTDS_LINK_ELEMENT (bin->fakesink_queue, bin->fakesink);

  if (config->dewarper_config.enable) {
    gst_bin_add_many (GST_BIN (bin->bin), bin->dewarper_bin.bin, NULL);
    NVGSTDS_LINK_ELEMENT (bin->tee, bin->dewarper_bin.bin);
    NVGSTDS_LINK_ELEMENT (bin->dewarper_bin.bin, bin->cap_filter);
  } else {
    link_element_to_tee_src_pad (bin->tee, bin->cap_filter);
  }

  NVGSTDS_LINK_ELEMENT (bin->cap_filter, bin->dec_que);
  NVGSTDS_LINK_ELEMENT (bin->dec_que, bin->nvvidconv);
  NVGSTDS_LINK_ELEMENT (bin->nvvidconv, bin->cap_filter1);

  link_element_to_tee_src_pad (bin->tee, bin->fakesink_queue);

  g_object_set (G_OBJECT (bin->fakesink), "sync", FALSE, "async", FALSE, NULL);
  g_object_set (G_OBJECT (bin->fakesink), "enable-last-sample", FALSE, NULL);

  ret = TRUE;

  g_timeout_add (1000, watch_source_status, bin);

  GST_CAT_DEBUG (NVDS_APP,
      "Decode bin created. Waiting for a new pad from decodebin to link");

done:

  if (!ret) {
    NVGSTDS_ERR_MSG_V ("%s failed", __func__);
  }
  return ret;
}
      if ((i != bin->num_bins) &&
          (appCtx->config.multi_source_config[0].type == NV_DS_SOURCE_RTSP)) {

Please modify the above code, this code in /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-app/deepstream_app.c.

Thank you for all the help @junshengy the reconnect now works for http streams and the pipeline doesn’t exit on EOS.

For anyone that is looking how to integrate the reconnect here are the changed functions for
deepstream_source_bin.c:

static gboolean
create_uridecode_src_bin_audio (NvDsSourceConfig * config, NvDsSrcBin * bin)
{
  gboolean ret = FALSE;
  guint const MAX_CAPS_LEN = 256;
  gchar caps_audio_resampler[MAX_CAPS_LEN];
  GstCaps *caps = NULL;
  bin->config = config;

  bin->src_elem = gst_element_factory_make (NVDS_ELEM_SRC_URI, "src_elem");
  if (!bin->src_elem) {
    NVGSTDS_ERR_MSG_V ("Could not create element 'src_elem'");
    goto done;
  }
  bin->latency = config->latency;
  bin->udp_buffer_size = config->udp_buffer_size;

  if (g_strrstr (config->uri, "file:/")) {
    config->live_source = FALSE;
  }
  if (g_strrstr (config->uri, "rtsp://") == config->uri) {
    configure_source_for_ntp_sync (bin->src_elem);
  }

  g_object_set (G_OBJECT (bin->src_elem), "uri", config->uri, NULL);
  g_signal_connect (G_OBJECT (bin->src_elem), "pad-added",
      G_CALLBACK (cb_newpad_audio), bin);


  bin->audio_converter =
      gst_element_factory_make (NVDS_ELEM_AUDIO_CONV, "audioconv_elem");
  if (!bin->audio_converter) {
    NVGSTDS_ERR_MSG_V ("Could not create element audio_converter");
    goto done;
  }

  bin->audio_resample =
      gst_element_factory_make (NVDS_ELEM_AUDIO_RESAMPLER,
      "audioresampler_elem");
  if (!bin->audio_resample) {
    NVGSTDS_ERR_MSG_V ("Could not create element audio_resample");
    goto done;
  }

  bin->cap_filter =
      gst_element_factory_make (NVDS_ELEM_CAPS_FILTER,
      "src_cap_filter_audioresample");
  if (!bin->cap_filter) {
    NVGSTDS_ERR_MSG_V ("Could not create src_cap_filter_audioresample");
    goto done;
  }

  if (snprintf (caps_audio_resampler, MAX_CAPS_LEN, "audio/x-raw, rate=%d",
          config->input_audio_rate)
      <= 0) {
    NVGSTDS_ERR_MSG_V ("Could not create caps to force rate=%d",
        config->input_audio_rate);
    goto done;
  }
  caps = gst_caps_from_string (caps_audio_resampler);
  g_object_set (G_OBJECT (bin->cap_filter), "caps", caps, NULL);
  gst_caps_unref (caps);


  gst_bin_add_many (GST_BIN (bin->bin), bin->src_elem,
      bin->audio_converter, bin->audio_resample, bin->cap_filter, NULL);

  NVGSTDS_LINK_ELEMENT (bin->audio_converter, bin->audio_resample);
  NVGSTDS_LINK_ELEMENT (bin->audio_resample, bin->cap_filter);

  NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->cap_filter, "src");

  ret = TRUE;

  GST_CAT_DEBUG (NVDS_APP,
      "Decode bin created. Waiting for a new pad from decodebin to link");

done:

  if (!ret) {
    NVGSTDS_ERR_MSG_V ("%s failed", __func__);
  }
  return ret;
}
static void
check_rtsp_reconnection_attempts (NvDsSrcBin * src_bin)
{
  gboolean remove_probe = TRUE;
  guint i = 0;
  for (i = 0; i < src_bin->parent_bin->num_bins; i++) {
    if (src_bin->parent_bin->sub_bins[i].config->type != NV_DS_SOURCE_RTSP || src_bin->parent_bin->sub_bins[i].config->type != NV_DS_SOURCE_URI)
      continue;
    if (src_bin->parent_bin->sub_bins[i].have_eos &&
        (src_bin->parent_bin->sub_bins[i].rtsp_reconnect_interval_sec == 0 ||
         src_bin->parent_bin->sub_bins[i].rtsp_reconnect_attempts == 0)) {
      remove_probe = FALSE;
      break;
    }
    if (src_bin->parent_bin->sub_bins[i].num_rtsp_reconnects <=
        src_bin->parent_bin->sub_bins[i].rtsp_reconnect_attempts) {
      if (src_bin->parent_bin->sub_bins[i].rtsp_reconnect_interval_sec ||
          !src_bin->parent_bin->sub_bins[i].have_eos) {
        remove_probe = FALSE;
        break;
      }
    }
  }

  if (remove_probe) {
    GstElement *pipeline =
        GST_ELEMENT_PARENT (GST_ELEMENT_PARENT (src_bin->bin));
    NVGSTDS_ELEM_REMOVE_PROBE (src_bin->parent_bin->
        nvstreammux_eosmonitor_probe, src_bin->parent_bin->streammux, "src");
    GST_ELEMENT_ERROR (pipeline, STREAM, FAILED,
        ("Reconnection attempts exceeded for all sources or EOS received."
            " Stopping pipeline"), (NULL));
  }
}

For deepstream_app.c:

if ((i != bin->num_bins) &&
          (appCtx->config.multi_source_config[0].type == NV_DS_SOURCE_RTSP || appCtx->config.multi_source_config[0].type == NV_DS_SOURCE_URI)) {
        // Error from one of RTSP source.
        NvDsSrcBin *subBin = &bin->sub_bins[i];