Gst for merging 4 webcams sometimes works, and sometimes it doesn't

Hello!

I´m trying this:


gst-launch-1.0 nvcompositor name=comp sink_0::xpos=0    sink_0::ypos=0    sink_0::width=1280 sink_0::height=720 \
                       sink_1::xpos=1280 sink_1::ypos=0    sink_1::width=1280 sink_1::height=720 \
                       sink_2::xpos=0    sink_2::ypos=720 sink_2::width=1280 sink_2::height=720 \
                       sink_3::xpos=1280 sink_3::ypos=720 sink_3::width=1280 sink_3::height=720 \
! nvvidconv \
! 'video/x-raw(memory:NVMM), width=2560, height=1440, framerate=60/1, format=NV12' \
! nvvidconv ! omxh264enc control-rate=2 bitrate=10000000 ! queue ! mux. alsasrc device='hw:2,0' ! audio/x-raw,width=16,depth=16,rate=32000,channels=2 ! queue ! audioconvert ! audioresample ! voaacenc ! aacparse ! qtmux name=mux ! filesink location=/home/ances/Desktop/0.mp4  -e \
v4l2src device=/dev/video0 ! image/jpeg,format=MJPG,width=1280,height=720,framerate=60/1 ! nvv4l2decoder mjpeg=1 ! nvvidconv ! 'video/x-raw(memory:NVMM),format=RGBA' ! queue ! comp.sink_0 \
v4l2src device=/dev/video1 ! image/jpeg,format=MJPG,width=1280,height=720,framerate=60/1 ! nvv4l2decoder mjpeg=1 ! nvvidconv ! 'video/x-raw(memory:NVMM),format=RGBA' ! queue ! comp.sink_1 \
v4l2src device=/dev/video2 ! image/jpeg,format=MJPG,width=1280,height=720,framerate=60/1 ! nvv4l2decoder mjpeg=1 ! nvvidconv ! 'video/x-raw(memory:NVMM),format=RGBA' ! queue ! comp.sink_2 \
v4l2src device=/dev/video3 ! image/jpeg,format=MJPG,width=1280,height=720,framerate=60/1 ! nvv4l2decoder mjpeg=1  ! nvvidconv ! 'video/x-raw(memory:NVMM),format=RGBA' ! queue ! comp.sink_3


In sometimes, work perfetly:

Setting pipeline to PAUSED ...
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Pipeline is live and does not need PREROLL ...
Setting pipeline to PLAYING ...
NvMMLiteOpen : Block : BlockType = 277
NVMEDIA: Reading vendor.tegra.display-size : status: 6
NvMMLiteBlockCreate : Block : BlockType = 277
NvMMLiteOpen : Block : BlockType = 277
NVMEDIA: Reading vendor.tegra.display-size : status: 6
NvMMLiteOpen : Block : BlockType = 277
NvMMLiteBlockCreate : Block : BlockType = 277
NVMEDIA: Reading vendor.tegra.display-size : status: 6
NvMMLiteOpen : Block : BlockType = 277
NVMEDIA: Reading vendor.tegra.display-size : status: 6
NvMMLiteBlockCreate : Block : BlockType = 277
NvMMLiteBlockCreate : Block : BlockType = 277
New clock: GstAudioSrcClock
Redistribute latency...
Framerate set to : 60 at NvxVideoEncoderSetParameterNvMMLiteOpen : Block : BlockType = 4
===== NVMEDIA: NVENC =====
NvMMLiteBlockCreate : Block : BlockType = 4
H264: Profile = 66, Level = 40
Redistribute latency...

But in others, I have this:

Setting pipeline to PAUSED ...
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Pipeline is live and does not need PREROLL ...
Setting pipeline to PLAYING ...
NvMMLiteOpen : Block : BlockType = 277
NVMEDIA: Reading vendor.tegra.display-size : status: 6
NvMMLiteBlockCreate : Block : BlockType = 277
NvMMLiteOpen : Block : BlockType = 277
NvMMLiteOpen : Block : BlockType = 277
NVMEDIA: Reading vendor.tegra.display-size : status: 6
NVMEDIA: Reading vendor.tegra.display-size : status: 6
NvMMLiteBlockCreate : Block : BlockType = 277
NvMMLiteBlockCreate : Block : BlockType = 277
NvMMLiteOpen : Block : BlockType = 277
NVMEDIA: Reading vendor.tegra.display-size : status: 6
NvMMLiteBlockCreate : Block : BlockType = 277
Redistribute latency...
New clock: GstAudioSrcClock
Framerate set to : 60 at NvxVideoEncoderSetParameterNvMMLiteOpen : Block : BlockType = 4
===== NVMEDIA: NVENC =====
NvMMLiteBlockCreate : Block : BlockType = 4
H264: Profile = 66, Level = 40
Redistribute latency...
Framerate set to : 60 at NvxVideoEncoderSetParameter

And it comes to a standstill.

Perhaps, Is this a memory problem?

Is there maybe a way to delay the power on of each camera? or some routine that allows to free memory just before executing this?

Thanks.

Hi,
Please eliminate alsasrc plugin to try video-only case. See if it works by removing audio part.

And we have deprecated omx plugins. Please replace omxh264enc with nvv4l2h264enc and try again.

I´ve tryed with this:

gst-launch-1.0 nvcompositor name=comp sink_0::xpos=0    sink_0::ypos=0    sink_0::width=1280 sink_0::height=720 \
                       sink_1::xpos=1280 sink_1::ypos=0    sink_1::width=1280 sink_1::height=720 \
                       sink_2::xpos=0    sink_2::ypos=720 sink_2::width=1280 sink_2::height=720 \
                       sink_3::xpos=1280 sink_3::ypos=720 sink_3::width=1280 sink_3::height=720 \
! nvvidconv \
! 'video/x-raw(memory:NVMM), width=2560, height=1440, framerate=60/1, format=NV12' \
! nvv4l2h264enc bitrate=10000000 !  h264parse ! matroskamux ! filesink location=/home/ances/Desktop/0.mp4  -e \
v4l2src device=/dev/video3 ! image/jpeg,format=MJPG,width=1280,height=720,framerate=60/1 ! nvv4l2decoder mjpeg=1 ! nvvidconv ! 'video/x-raw(memory:NVMM),format=RGBA' ! queue ! comp.sink_0 \
v4l2src device=/dev/video2 ! image/jpeg,format=MJPG,width=1280,height=720,framerate=60/1 ! nvv4l2decoder mjpeg=1 ! nvvidconv ! 'video/x-raw(memory:NVMM),format=RGBA' ! queue ! comp.sink_1 \
v4l2src device=/dev/video1 ! image/jpeg,format=MJPG,width=1280,height=720,framerate=60/1 ! nvv4l2decoder mjpeg=1 ! nvvidconv ! 'video/x-raw(memory:NVMM),format=RGBA' ! queue ! comp.sink_2 \
v4l2src device=/dev/video0 ! image/jpeg,format=MJPG,width=1280,height=720,framerate=60/1 ! nvv4l2decoder mjpeg=1  ! nvvidconv ! 'video/x-raw(memory:NVMM),format=RGBA' ! queue ! comp.sink_3

And the problem is the same: Sometimes it works, sometimes it doesn’t.

The error displayed is:

Setting pipeline to PAUSED ...
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Opening in BLOCKING MODE
Pipeline is live and does not need PREROLL ...
Setting pipeline to PLAYING ...
New clock: GstSystemClock
NvMMLiteOpen : Block : BlockType = 277
NvMMLiteOpen : Block : BlockType = 277
NVMEDIA: Reading vendor.tegra.display-size : status: 6
NVMEDIA: Reading vendor.tegra.display-size : status: 6
NvMMLiteBlockCreate : Block : BlockType = 277
NvMMLiteOpen : Block : BlockType = 277
NVMEDIA: Reading vendor.tegra.display-size : status: 6
NvMMLiteOpen : Block : BlockType = 277
NVMEDIA: Reading vendor.tegra.display-size : status: 6
NvMMLiteBlockCreate : Block : BlockType = 277
NvMMLiteBlockCreate : Block : BlockType = 277
NvMMLiteBlockCreate : Block : BlockType = 277
Redistribute latency...
NvMMLiteOpen : Block : BlockType = 4
===== NVMEDIA: NVENC =====
NvMMLiteBlockCreate : Block : BlockType = 4
Redistribute latency...
H264: Profile = 66, Level = 0
Redistribute latency...
NvMMLiteOpen : Block : BlockType = 4
===== NVMEDIA: NVENC =====
NvMMLiteBlockCreate : Block : BlockType = 4
H264: Profile = 66, Level = 0
ERROR: from element /GstPipeline:pipeline0/GstMatroskaMux:matroskamux0: Could not multiplex stream.
Additional debug info:
matroska-mux.c(986): gst_matroska_mux_video_pad_setcaps (): /GstPipeline:pipeline0/GstMatroskaMux:matroskamux0:
Caps changes are not supported by Matroska
EOS on shutdown enabled -- waiting for EOS after Error
Waiting for EOS...
ERROR: from element /GstPipeline:pipeline0/GstMatroskaMux:matroskamux0: Could not multiplex stream.
Additional debug info:
matroska-mux.c(986): gst_matroska_mux_video_pad_setcaps (): /GstPipeline:pipeline0/GstMatroskaMux:matroskamux0:
Caps changes are not supported by Matroska
ERROR: from element /GstPipeline:pipeline0/GstMatroskaMux:matroskamux0: Could not multiplex stream.
Additional debug info:
matroska-mux.c(986): gst_matroska_mux_video_pad_setcaps (): /GstPipeline:pipeline0/GstMatroskaMux:matroskamux0:
Caps changes are not supported by Matroska
ERROR: from element /GstPipeline:pipeline0/GstV4l2Src:v4l2src0: Internal data stream error.
Additional debug info:
gstbasesrc.c(3055): gst_base_src_loop (): /GstPipeline:pipeline0/GstV4l2Src:v4l2src0:
streaming stopped, reason not-negotiated (-4)
ERROR: from element /GstPipeline:pipeline0/GstQueue:queue0: Internal data stream error.
Additional debug info:
gstqueue.c(988): gst_queue_handle_sink_event (): /GstPipeline:pipeline0/GstQueue:queue0:
streaming stopped, reason not-negotiated (-4)
ERROR: from element /GstPipeline:pipeline0/GstV4l2Src:v4l2src1: Internal data stream error.
Additional debug info:
gstbasesrc.c(3055): gst_base_src_loop (): /GstPipeline:pipeline0/GstV4l2Src:v4l2src1:
streaming stopped, reason not-negotiated (-4)
ERROR: from element /GstPipeline:pipeline0/GstQueue:queue1: Internal data stream error.
Additional debug info:
gstqueue.c(988): gst_queue_handle_sink_event (): /GstPipeline:pipeline0/GstQueue:queue1:
streaming stopped, reason not-negotiated (-4)
ERROR: from element /GstPipeline:pipeline0/GstV4l2Src:v4l2src2: Internal data stream error.
Additional debug info:
gstbasesrc.c(3055): gst_base_src_loop (): /GstPipeline:pipeline0/GstV4l2Src:v4l2src2:
streaming stopped, reason not-negotiated (-4)
ERROR: from element /GstPipeline:pipeline0/GstQueue:queue2: Internal data stream error.
Additional debug info:
gstqueue.c(988): gst_queue_handle_sink_event (): /GstPipeline:pipeline0/GstQueue:queue2:
streaming stopped, reason not-negotiated (-4)
ERROR: from element /GstPipeline:pipeline0/GstV4l2Src:v4l2src3: Internal data stream error.
Additional debug info:
gstbasesrc.c(3055): gst_base_src_loop (): /GstPipeline:pipeline0/GstV4l2Src:v4l2src3:
streaming stopped, reason not-negotiated (-4)
ERROR: from element /GstPipeline:pipeline0/GstQueue:queue3: Internal data stream error.
Additional debug info:
gstqueue.c(988): gst_queue_handle_sink_event (): /GstPipeline:pipeline0/GstQueue:queue3:
streaming stopped, reason not-negotiated (-4)

With filesrc always works perfectly.

I think it has something to do with some “peak” that occurs when the 4 cameras are opened at the same time. I had similar errors with python, making a stream for each camera, opening them all at once (and I “fix” it letting delay between each opening)

That is why I made the suggestion to open the cameras with a certain delay between them.

I have tried lowering the resolution, or the FPS, and the problem persists.

Hi,
The nvcompositor plugin is open source from r32.5.1. Please apply this patch:

diff --git a/gst-nvcompositor/gstnvcompositor.c b/gst-nvcompositor/gstnvcompositor.c
index d0c23d7..c85c515 100644
--- a/gst-nvcompositor/gstnvcompositor.c
+++ b/gst-nvcompositor/gstnvcompositor.c
@@ -708,7 +708,9 @@ gst_nvcompositor_pad_init (GstNvCompositorPad * nvcompo_pad)
 enum
 {
   PROP_0,
-  PROP_BACKGROUND
+  PROP_BACKGROUND,
+  PROP_BACKGROUND_WIDTH,
+  PROP_BACKGROUND_HEIGHT
 };
 
 #define GST_TYPE_NVCOMPOSITOR_BACKGROUND (gst_nvcompositor_background_get_type())
@@ -784,6 +786,12 @@ gst_nvcompositor_set_property (GObject * object,
 //    gst_nvcompositor_parse_bgcolor (value, nvcomp);
       nvcomp->background = g_value_get_enum (value);
       break;
+    case PROP_BACKGROUND_WIDTH:
+      nvcomp->bg_width = g_value_get_int(value);
+      break;
+    case PROP_BACKGROUND_HEIGHT:
+      nvcomp->bg_height = g_value_get_int(value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -808,6 +816,12 @@ gst_nvcompositor_get_property (GObject * object,
 //    gst_nvcompositor_get_bgcolor (value, nvcomp);
       g_value_set_enum (value, nvcomp->background);
       break;
+    case PROP_BACKGROUND_WIDTH:
+      g_value_set_int(value, nvcomp->bg_width);
+      break;
+    case PROP_BACKGROUND_HEIGHT:
+      g_value_set_int(value, nvcomp->bg_height);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -837,6 +851,7 @@ gst_nvcompositor_fixate_caps (GstAggregator * agg, GstCaps * caps)
   gint suitable_fps_n = -1, suitable_fps_d = -1;
   gint suitable_width = -1, suitable_height = -1;
   GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
+  GstNvCompositor *nvcomp = GST_NVCOMPOSITOR (vagg);
 
   ret = gst_caps_make_writable (caps);
 
@@ -897,6 +912,10 @@ gst_nvcompositor_fixate_caps (GstAggregator * agg, GstCaps * caps)
     suitable_fps_n = 30;
     suitable_fps_d = 1;
   }
+  if (nvcomp->bg_width > 0 && nvcomp->bg_height > 0) {
+      suitable_width = nvcomp->bg_width;
+      suitable_height = nvcomp->bg_height;
+  }
 
   gst_structure_fixate_field_nearest_fraction (str, "framerate", suitable_fps_n,
       suitable_fps_d);
@@ -1421,6 +1440,15 @@ gst_nvcompositor_class_init (GstNvCompositorClass * klass)
           GST_TYPE_NVCOMPOSITOR_BACKGROUND,
           DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
+  g_object_class_install_property (gobject_class, PROP_BACKGROUND_WIDTH,
+      g_param_spec_int ("background-w", "Width", "Width of the background",
+          0, G_MAXINT, 0,
+          G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
+  g_object_class_install_property (gobject_class, PROP_BACKGROUND_HEIGHT,
+      g_param_spec_int ("background-h", "Height", "Height of the background",
+          0, G_MAXINT, 0,
+          G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
+
 /* TODO: Replace background property static enum with rgb boxed array */
 #if 0
   g_object_class_install_property (gobject_class, PROP_BACKGROUND,
@@ -1457,6 +1485,8 @@ gst_nvcompositor_init (GstNvCompositor * nvcomp)
   nvcomp->bg.g = 0;
   nvcomp->bg.b = 0;
   nvcomp->pool = NULL;
+  nvcomp->bg_width = 0;
+  nvcomp->bg_height = 0;
 }
 
 /* NvCompositor Element registration */
diff --git a/gst-nvcompositor/gstnvcompositor.h b/gst-nvcompositor/gstnvcompositor.h
index 60caa5c..90ad042 100644
--- a/gst-nvcompositor/gstnvcompositor.h
+++ b/gst-nvcompositor/gstnvcompositor.h
@@ -114,6 +114,8 @@ struct _GstNvCompositor
 
   GstNvCompBgcolor bg;
   GstNvCompositorBackground background;
+  gint bg_width;
+  gint bg_height;
 
   gboolean nvcomppool;
   GstBufferPool *pool;

And rebuild/replace the prebuilt lib. So that you can set background width and height. Please set the properties and try again.

Here is a related topic for reference:
Gstreamer using nvcomposer to filesink hangs, but nvcomposer to nvoverlaysink works fine - #19 by DaneLLL

Now this works perfectly always.

Thanks a lot!