source (IP camera (Komoto L6s_10180p)) disconnection!!

Hi
I am testing the deepstream for source disconnection. I disconnect the camera however when I reconnect the camera the software crash (I am running deepstream-app).
I came across the function:
“static gboolean watch_source_status (gpointer data)”
which seems to be responsible for resource re-connection.
any idea why the software still crashing?

Could you check the core dump file and the back trace to see where the app crashed.

Hi bcao
this is the message that i am getting:

WARNING from source: Could not read from resource.
Debug info: gstrtspsrc.c(5293): gst_rtspsrc_loop_udp (): /GstPipeline:pipeline/GstBin:multi_src_bin/GstBin:src_sub_bin1/GstURIDecodeBin:src_elem/GstRTSPSrc:source:
Unhandled return value -7.
ERROR from source: Could not read from resource.
Debug info: gstrtspsrc.c(5361): gst_rtspsrc_loop_udp (): /GstPipeline:pipeline/GstBin:multi_src_bin/GstBin:src_sub_bin1/GstURIDecodeBin:src_elem/GstRTSPSrc:source:
Could not receive message. (System error)
2ERROR from source: Internal data stream error.
Debug info: gstrtspsrc.c(5653): gst_rtspsrc_loop (): /GstPipeline:pipeline/GstBin:multi_src_bin/GstBin:src_sub_bin1/GstURIDecodeBin:src_elem/GstRTSPSrc:source:
streaming stopped, reason error (-5)
2Quitting
App run failed

in the function:

static gboolean bus_callback (GstBus * bus, GstMessage * message, gpointer data)
in deepstream_app.c

I tried to trace back the issue but not succeed.

Per the log on comment #3, I didn’t see there is a crash for ds-app.
Could you refer https://docs.nvidia.com/metropolis/deepstream/4.0/dev-guide/index.html#page/DeepStream_Development_Guide%2Fdeepstream_app_config.3.1.html to check your config and would you mind to share your change if you change the ds-app source code?

hi bcao
thx for reply.
I to make deepstream able to restart the rtsp source (IP camera) in case of disconnection. if I turn off the camera and turn it on, the software terminate the whole instance once the camera back.
this is the only change i made in deepstream_app.c file.
I coment out these two lines that the software do not terminate other streams in case of single stream goes down.
now my question is how i can restart the streams that is reconnected?

//appCtx->return_value = -1;
//appCtx->quit = TRUE;

static gboolean
bus_callback (GstBus * bus, GstMessage * message, gpointer data)
{
AppCtx *appCtx = (AppCtx *) data;
GST_CAT_DEBUG (NVDS_APP,
“Received message on bus: source %s, msg_type %s”,
GST_MESSAGE_SRC_NAME (message), GST_MESSAGE_TYPE_NAME (message));
switch (GST_MESSAGE_TYPE (message)) {
case GST_MESSAGE_INFO:{
GError *error = NULL;
gchar *debuginfo = NULL;
gst_message_parse_info (message, &error, &debuginfo);
g_printerr (“INFO from %s: %s\n”,
GST_OBJECT_NAME (message->src), error->message);
if (debuginfo) {
g_printerr (“Debug info: %s\n”, debuginfo);
}
g_error_free (error);
g_free (debuginfo);
break;
}
case GST_MESSAGE_WARNING:{
GError *error = NULL;
gchar *debuginfo = NULL;
gst_message_parse_warning (message, &error, &debuginfo);
g_printerr (“WARNING from %s: %s\n”,
GST_OBJECT_NAME (message->src), error->message);
if (debuginfo) {
g_printerr (“Debug info: %s\n”, debuginfo);
}
g_error_free (error);
g_free (debuginfo);
break;
}
case GST_MESSAGE_ERROR:{
GError *error = NULL;
gchar *debuginfo = NULL;
guint i = 0;
gst_message_parse_error (message, &error, &debuginfo);
g_printerr (“ERROR from %s: %s\n”,
GST_OBJECT_NAME (message->src), error->message);
if (debuginfo) {
g_printerr (“Debug info: %s\n”, debuginfo);
}

  NvDsSrcParentBin *bin = &appCtx->pipeline.multi_src_bin;
  g_print("bin num:%d", bin->num_bins);
  for (i = 0; i < bin->num_bins; i++) {
    if (bin->sub_bins[i].src_elem == (GstElement *) GST_MESSAGE_SRC (message))
      break;
  }

  if ((i != bin->num_bins) &&
      (appCtx->config.multi_source_config[0].type == NV_DS_SOURCE_RTSP)) {
    // Error from one of RTSP source.
    NvDsSrcBin *subBin = &bin->sub_bins[i];

    if (!subBin->reconfiguring ||
        g_strrstr(debuginfo, "500 (Internal Server Error)")) {
      if (!subBin->reconfiguring) {
        // Check status of stream at regular interval.
        g_timeout_add (SOURCE_RESET_INTERVAL_IN_MS,
                       watch_source_status, subBin);
      }
      // Reconfigure the stream.
      subBin->reconfiguring = TRUE;
      g_timeout_add (20, reset_source_pipeline, subBin);
    }
    g_error_free (error);
    g_free (debuginfo);
    return TRUE;
  }

  g_error_free (error);
  g_free (debuginfo);
  //appCtx->return_value = -1;
  //appCtx->quit = TRUE;
  break;
}
case GST_MESSAGE_STATE_CHANGED:{
  GstState oldstate, newstate;
  gst_message_parse_state_changed (message, &oldstate, &newstate, NULL);
  if (GST_ELEMENT (GST_MESSAGE_SRC (message)) == appCtx->pipeline.pipeline) {
    switch (newstate) {
      case GST_STATE_PLAYING:
        NVGSTDS_INFO_MSG_V ("Pipeline running\n");
        GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS (GST_BIN (appCtx->
                pipeline.pipeline), GST_DEBUG_GRAPH_SHOW_ALL,
            "ds-app-playing");
        break;
      case GST_STATE_PAUSED:
        if (oldstate == GST_STATE_PLAYING) {
          NVGSTDS_INFO_MSG_V ("Pipeline paused\n");
        }
        break;
      case GST_STATE_READY:
        GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS (GST_BIN (appCtx->pipeline.
                pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "ds-app-ready");
        if (oldstate == GST_STATE_NULL) {
          NVGSTDS_INFO_MSG_V ("Pipeline ready\n");
        } else {
          NVGSTDS_INFO_MSG_V ("Pipeline stopped\n");
        }
        break;
      case GST_STATE_NULL:
        g_mutex_lock (&appCtx->app_lock);
        g_cond_broadcast (&appCtx->app_cond);
        g_mutex_unlock (&appCtx->app_lock);
        break;
      default:
        break;
    }
  }
  break;
}
case GST_MESSAGE_EOS:{
  /*
   * In normal scenario, this would use g_main_loop_quit() to exit the
   * loop and release the resources. Since this application might be
   * running multiple pipelines through configuration files, it should wait
   * till all pipelines are done.
   */
  NVGSTDS_INFO_MSG_V ("Received EOS. Exiting ...\n");
  appCtx->quit = TRUE;
  return FALSE;
  break;
}
default:
  break;

}
return TRUE;
}

Will check it and give your reply, BTW, it will be better if you can share you config file.

this is my config file.
Thanks for ur reply

https://filebin.net/lj7s0ljsbta6xtvp/deepstream_app_config_yoloV3.txt?t=jrc725ut

Hey customer, could you set the type=4 under source group, We support reconnection for type=4 only not type=3 (MultiURI).

hey bcao,
just to make it more clear, I am talking about the situation where the IP camera disconnected from internet or shut downed and again reconnected.
I did what you said and did not worked. when deepstream sense the camera back it terminate the whole streams. the picture below is what i am getting.

https://ibb.co/t8mKZwP

We had made a patch and are verifying it, will provide the patch with you after we verified it locally.

Thanks.

could you apply the patch and try again.

diff -u -r apps/apps-common/includes/deepstream_sources.h apps/apps-common/includes/deepstream_sources.h
--- apps/apps-common/includes/deepstream_sources.h	2019-12-02 15:41:49.767009929 +0530
+++ apps/apps-common/includes/deepstream_sources.h	2019-12-02 15:30:25.460908485 +0530
@@ -136,6 +136,7 @@
   guint num_fr_on;
   gboolean live_source;
   rtcp_sender_report_callback rtcp_sender_report_cb;
+  gulong nvstreammux_eosmonitor_probe;
 } NvDsSrcParentBin;


diff -u -r apps/apps-common/src/deepstream_source_bin.c apps/apps-common/src/deepstream_source_bin.c
--- apps/apps-common/src/deepstream_source_bin.c	2019-12-02 15:41:49.767009929 +0530
+++ apps/apps-common/src/deepstream_source_bin.c	2019-12-02 15:30:25.460908485 +0530
@@ -33,6 +33,8 @@
 GST_DEBUG_CATEGORY_EXTERN (NVDS_APP);
 GST_DEBUG_CATEGORY_EXTERN (APP_CFG_PARSER_CAT);

+static gboolean have_rtspsrc = FALSE;
+
 static gboolean
 set_camera_csi_params (NvDsSourceConfig * config, NvDsSrcBin * bin)
 {
@@ -456,6 +458,23 @@
   return ret;
 }

+/**
+ * Probe function to drop EOS events from nvstreammux when RTSP sources
+ * are being used so that app does not quit from EOS in case of RTSP
+ * connection errors and tries to reconnect.
+ */
+static GstPadProbeReturn
+nvstreammux_eosmonitor_probe_func (GstPad * pad, GstPadProbeInfo * info,
+    gpointer u_data)
+{
+  if (info->type & GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM) {
+    GstEvent *event = (GstEvent *) info->data;
+    if (GST_EVENT_TYPE (event) == GST_EVENT_EOS)
+      return GST_PAD_PROBE_DROP;
+  }
+  return GST_PAD_PROBE_OK;
+}
+
 static gboolean
 create_rtsp_src_bin (NvDsSourceConfig * config, NvDsSrcBin * bin)
 {
@@ -554,6 +573,8 @@
   GST_CAT_DEBUG (NVDS_APP,
       "Decode bin created. Waiting for a new pad from decodebin to link");

+  have_rtspsrc = TRUE;
+
 done:

   if (!ret) {
@@ -770,6 +791,13 @@

   NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->streammux, "src");

+  if (have_rtspsrc) {
+    NVGSTDS_ELEM_ADD_PROBE (bin->nvstreammux_eosmonitor_probe, bin->streammux,
+        "src", nvstreammux_eosmonitor_probe_func,
+        GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM,
+        bin);
+  }
+
   ret = TRUE;

 done:

reconnection (1).7z (1.04 KB)

hi bcao
I tested your patch but it don’t solve the issue. can you share your deepstream_source_bin.c file.

Are u using DS 4.0.1? see attachment for source code.
deepstream_source_bin.7z (6.87 KB)

yes i am using DS 4.0.1.
At which function DS frequently check if there is reconnected source?
since this issue happen for specific camera brand.

Could you give more details about the " specific camera brand." And could you share me the log if the issue still not solved?, thanks.

its komoto L6s_10180p for ANPR purpose.

https://filebin.net/08kh0fzsms8j2fgb/Screenshot_from_2019-12-04_11-17-07.png?t=3zspys1o

Ok, so you cannot repro the issue when using other IP camera, right? and would you mind to share the logs when you using the two types camera? BTW, did you remove the GST cache when applying the patch?

yes I clear the cache.
here is the log for disconnection of two different camera brand. I am printing out the message in “static gboolean bus_callback (GstBus * bus, GstMessage * message, gpointer data)” function in deepstream_app.c file.

g_print(“%s\n”, GST_MESSAGE_TYPE_NAME (message));

https://filebin.net/km65kg6vqoj7r8oq/outputLog.txt?t=3zh1vulc