After the agx orin is powered on, the first rtmp push multiplexes are stuck

I encountered a problem. After the agx orin platform was powered on and started, I got stuck when I used gst-launch-1.0 to splice the four-way and rtmp push streams at the same time for the first time. I’m not sure why this is, but if there’s only one flow, it works, and then after you’ve successfully pushed the flow all the way, all four will work.

gst-launch-1.0 nvcompositor name=comp sink_0::xpos=0 sink_0::ypos=0 sink_0::width=960 sink_0::height=540 \
>         sink_1::xpos=960 sink_1::ypos=0 sink_1::width=960 sink_1::height=540 \
>         sink_2::xpos=0 sink_2::ypos=540 sink_2::width=960 sink_2::height=540 \
>         sink_3::xpos=960 sink_3::ypos=540 sink_3::width=960 sink_3::height=540 \
>         ! nvvidconv ! 'video/x-raw(memory:NVMM),width=1920,height=1080,format=NV12,framerate=10/1' ! nvv4l2h264enc  ! h264parse ! flvmux ! rtmpsink blocksize=24834048  location='rtmp://127.0.0.1/live/mux4' \
>         nvarguscamerasrc sensor-id=0 wbmode=0  saturation=0 blocksize=124170240 ! 'video/x-raw(memory:NVMM), width=(int)4128, height=(int)3008,format=(string)NV12, framerate=(fraction)10/1' ! tee name=cam0 ! \
>        queue !  nvvidconv  ! "video/x-raw(memory:NVMM), width=2064, height=1504, format=(string)NV12" !  nvvidconv  ! "video/x-raw" !  multifilesink max-file-size=1862553600  next-file=4 location=cam0_%04d.nv12 -e cam0. ! queue ! comp.sink_0 cam0. ! queue !   nvvidconv ! 'video/x-raw(memory:NVMM),width=3840,height=2160,format=NV12,framerate=10/1' ! nvv4l2h265enc bitrate=20000000 maxperf-enable=true ! h265parse ! mp4mux ! filesink location=cam-0.mp4 \
>         nvarguscamerasrc sensor-id=1 wbmode=0  saturation=0 blocksize=124170240 ! 'video/x-raw(memory:NVMM), width=(int)4128, height=(int)3008,format=(string)NV12, framerate=(fraction)10/1' ! tee name=cam1 ! \
>         queue !  nvvidconv  ! "video/x-raw(memory:NVMM), width=2064, height=1504, format=(string)NV12" !  nvvidconv  ! "video/x-raw" !  multifilesink max-file-size=1862553600  next-file=4 location=cam1_%04d.nv12 -e cam1. ! queue ! comp.sink_1  cam1. ! queue !   nvvidconv ! 'video/x-raw(memory:NVMM),width=3840,height=2160,format=NV12,framerate=10/1'  ! nvv4l2h265enc bitrate=20000000 maxperf-enable=true ! h265parse ! mp4mux ! filesink location=cam-1.mp4 \
>         nvarguscamerasrc sensor-id=2 wbmode=0  saturation=0 blocksize=124170240 ! 'video/x-raw(memory:NVMM), width=(int)4128, height=(int)3008,format=(string)NV12, framerate=(fraction)10/1' ! tee name=cam2 ! \
>        queue !  nvvidconv  ! "video/x-raw(memory:NVMM), width=2064, height=1504, format=(string)NV12" !  nvvidconv  ! "video/x-raw" !  multifilesink max-file-size=1862553600  next-file=4 location=cam2_%04d.nv12 -e cam2. ! queue ! comp.sink_2  cam2. ! queue !   nvvidconv ! 'video/x-raw(memory:NVMM),width=3840,height=2160,format=NV12,framerate=10/1' ! nvv4l2h265enc bitrate=20000000 maxperf-enable=true ! h265parse ! mp4mux ! filesink location=cam-2.mp4 \
>         nvarguscamerasrc sensor-id=3 wbmode=0  saturation=0 blocksize=124170240 ! 'video/x-raw(memory:NVMM), width=(int)4128, height=(int)3008,format=(string)NV12, framerate=(fraction)10/1' ! tee name=cam3 ! \
>        queue !  nvvidconv  ! "video/x-raw(memory:NVMM), width=2064, height=1504, format=(string)NV12" !  nvvidconv  ! "video/x-raw" !  multifilesink max-file-size=1862553600  next-file=4 location=cam3_%04d.nv12 -e cam3. ! queue ! comp.sink_3  cam3. ! queue !   nvvidconv ! 'video/x-raw(memory:NVMM),width=3840,height=2160,format=NV12,framerate=10/1'  ! nvv4l2h265enc bitrate=20000000 maxperf-enable=true ! h265parse ! mp4mux ! filesink location=cam-3.mp4
Setting pipeline to PAUSED ...
Opening in BLOCKING MODE 
Opening in BLOCKING MODE 
Opening in BLOCKING MODE 
Opening in BLOCKING MODE 
Opening in BLOCKING MODE 
Pipeline is live and does not need PREROLL ...
Setting pipeline to PLAYING ...
New clock: GstSystemClock
Caught SIGSEGV
NvMMLiteOpen : Block : BlockType = 8 
Redistribute latency...
===== NVMEDIA: NVENC =====
NvMMLiteBlockCreate : Block : BlockType = 8 
NvMMLiteOpen : Block : BlockType = 8 
===== NVMEDIA: NVENC =====
NvMMLiteBlockCreate : Block : BlockType = 8 
NvMMLiteOpen : Block : BlockType = 8 
===== NVMEDIA: NVENC =====
NvMMLiteBlockCreate : Block : BlockType = 8 
NvMMLiteOpen : Block : BlockType = 8 
===== NVMEDIA: NVENC =====
NvMMLiteBlockCreate : Block : BlockType = 8 
Spinning.  Please run 'gdb gst-launch-1.0 2988' to continue debugging, Ctrl-C to quit, or Ctrl-\ to dump core.
GST_ARGUS: Creating output stream
GST_ARGUS: Creating output stream
CONSUMER: Waiting until producer is connected...
GST_ARGUS: Creating output stream
CONSUMER: Waiting until producer is connected...
GST_ARGUS: Creating output stream
CONSUMER: Waiting until producer is connected...
CONSUMER: Waiting until producer is connected...
GST_ARGUS: Available Sensor modes :
GST_ARGUS: 4128 x 3008 FR = 28.999999 fps Duration = 34482760 ; Analog Gain range min 1.000000, max 251.188705; Exposure Range min 11000, max 660000000;

GST_ARGUS: Running with following settings:
   Camera index = 2 
   Camera mode  = 0 
   Output Stream W = 4128 H = 3008 
   seconds to Run    = 0 
   Frame Rate = 28.999999 
GST_ARGUS: Available Sensor modes :
GST_ARGUS: Setup Complete, Starting captures for 0 seconds
GST_ARGUS: Starting repeat capture requests.
GST_ARGUS: 4128 x 3008 FR = 28.999999 fps Duration = 34482760 ; Analog Gain range min 1.000000, max 251.188705; Exposure Range min 11000, max 660000000;

GST_ARGUS: Running with following settings:
   Camera index = 1 
   Camera mode  = 0 
   Output Stream W = 4128 H = 3008 
   seconds to Run    = 0 
   Frame Rate = 28.999999 
GST_ARGUS: Available Sensor modes :
GST_ARGUS: Setup Complete, Starting captures for 0 seconds
GST_ARGUS: Starting repeat capture requests.
GST_ARGUS: 4128 x 3008 FR = 28.999999 fps Duration = 34482760 ; Analog Gain range min 1.000000, max 251.188705; Exposure Range min 11000, max 660000000;

GST_ARGUS: Running with following settings:
   Camera index = 0 
   Camera mode  = 0 
   Output Stream W = 4128 H = 3008 
   seconds to Run    = 0 
   Frame Rate = 28.999999 
GST_ARGUS: Available Sensor modes :
GST_ARGUS: Setup Complete, Starting captures for 0 seconds
GST_ARGUS: Starting repeat capture requests.
GST_ARGUS: 4128 x 3008 FR = 28.999999 fps Duration = 34482760 ; Analog Gain range min 1.000000, max 251.188705; Exposure Range min 11000, max 660000000;

GST_ARGUS: Running with following settings:
   Camera index = 3 
   Camera mode  = 0 
   Output Stream W = 4128 H = 3008 
   seconds to Run    = 0 
   Frame Rate = 28.999999 
GST_ARGUS: Setup Complete, Starting captures for 0 seconds
GST_ARGUS: Starting repeat capture requests.
CONSUMER: Producer has connected; continuing.
CONSUMER: Producer has connected; continuing.
CONSUMER: Producer has connected; continuing.
CONSUMER: Producer has connected; continuing.
Acquired Frame: 10, time sec 0 msec 897 id:281469704159712
Acquired Frame: 10, time sec 0 msec 903 id:281469314384352
Acquired Frame: 10, time sec 0 msec 910 id:281469339562464
Acquired Frame: 10, time sec 0 msec 917 id:281469729337824
Acquired Frame: 11, time sec 0 msec 995 id:281469314384352
Acquired Frame: 11, time sec 1 msec 9 id:281469704159712
Acquired Frame: 11, time sec 1 msec 16 id:281469339562464
Acquired Frame: 11, time sec 1 msec 30 id:281469729337824
Acquired Frame: 12, time sec 1 msec 97 id:281469729337824
Acquired Frame: 12, time sec 1 msec 103 id:281469704159712
Acquired Frame: 12, time sec 1 msec 111 id:281469314384352
Acquired Frame: 12, time sec 1 msec 117 id:281469339562464
Acquired Frame: 13, time sec 1 msec 198 id:281469729337824
Acquired Frame: 13, time sec 1 msec 203 id:281469314384352
Acquired Frame: 13, time sec 1 msec 209 id:281469704159712
Acquired Frame: 13, time sec 1 msec 218 id:281469339562464
Acquired Frame: 14, time sec 1 msec 296 id:281469339562464
Acquired Frame: 14, time sec 1 msec 303 id:281469704159712
Acquired Frame: 14, time sec 1 msec 309 id:281469729337824
Acquired Frame: 14, time sec 1 msec 316 id:281469314384352
Acquired Frame: 15, time sec 1 msec 397 id:281469729337824
Acquired Frame: 15, time sec 1 msec 402 id:281469704159712
Acquired Frame: 15, time sec 1 msec 410 id:281469314384352
Acquired Frame: 15, time sec 1 msec 416 id:281469339562464
Acquired Frame: 16, time sec 1 msec 496 id:281469314384352
Acquired Frame: 16, time sec 1 msec 502 id:281469729337824
Acquired Frame: 16, time sec 1 msec 509 id:281469339562464
Acquired Frame: 16, time sec 1 msec 517 id:281469704159712
Acquired Frame: 17, time sec 1 msec 596 id:281469314384352
Acquired Frame: 17, time sec 1 msec 603 id:281469729337824
Acquired Frame: 17, time sec 1 msec 618 id:281469339562464
Acquired Frame: 17, time sec 1 msec 609 id:281469704159712
Acquired Frame: 18, time sec 1 msec 697 id:281469314384352
Acquired Frame: 18, time sec 1 msec 704 id:281469704159712
Acquired Frame: 18, time sec 1 msec 716 id:281469729337824
Acquired Frame: 18, time sec 1 msec 710 id:281469339562464

The above program runs the printed results, from 18 onwards will be stuck, and the previous data is not written to the file, only the file is created, the file size is 0.

Hi,
Please share which release version you are using. The latest release is Jetpack 5.1.2, If you use previous version, we would suggest upgrade to latest version and try.

1 Like

hi,
At present, the project has reached the final stage, just this problem has not been solved, there is no better solution under this version.

xxx@xxx-desktop:~$ cat /etc/nv_tegra_release 
# R35 (release), REVISION: 3.1, GCID: 32827747, BOARD: t186ref, EABI: aarch64, DATE: Sun Mar 19 15:19:21 UTC 2023

Hi,
There is a patch for nvcompositor plugin if you use r35.3.1:
Jetson/L4T/r35.3.x patches - eLinux.org

[gstreamer] nvcompositor-sigsegv

Please apply it to rebuild the plugin and try.

And for further investigation, you may break down the commands:

... ! nvvidconv ! 'video/x-raw(memory:NVMM),width=1920,height=1080,format=NV12,framerate=10/1' ! fakesink
... ! nvvidconv ! 'video/x-raw(memory:NVMM),width=1920,height=1080,format=NV12,framerate=10/1' ! nvv4l2h264enc ! fakesink
... ! nvvidconv ! 'video/x-raw(memory:NVMM),width=1920,height=1080,format=NV12,framerate=10/1' ! nvv4l2h264enc ! h264parse ! flvmux !  fakesink

to get more information. And can add queue plugin for a try.

hi,
The failure of multiple channel push has been successfully solved. But now there is such a situation, or the same command, but sometimes there is an out-of-sync phenomenon, the test ten times may only be once out of sync, my multi-lens hardware triggered to synchronize.In addition, an MP4 file recorded in this way will appear two identical frames when converting all MP4 frames to jpg using ffmepg

Acquired Frame: 10, time sec 0 msec 898 id:281469893186016
Acquired Frame: 10, time sec 0 msec 904 id:281469943542240
Acquired Frame: 10, time sec 0 msec 910 id:281470238724576
Acquired Frame: 10, time sec 0 msec 918 id:281469918364128
Acquired Frame: 11, time sec 0 msec 997 id:281469893186016
Acquired Frame: 11, time sec 1 msec 18 id:281469943542240
Acquired Frame: 11, time sec 1 msec 53 id:281470238724576
Acquired Frame: 11, time sec 1 msec 75 id:281469918364128
Acquired Frame: 12, time sec 1 msec 99 id:281469893186016
Acquired Frame: 12, time sec 1 msec 118 id:281469943542240
Acquired Frame: 12, time sec 1 msec 129 id:281470238724576
Acquired Frame: 12, time sec 1 msec 134 id:281469918364128
Acquired Frame: 13, time sec 1 msec 205 id:281469943542240
Acquired Frame: 13, time sec 1 msec 198 id:281470238724576
Acquired Frame: 13, time sec 1 msec 212 id:281469918364128
Acquired Frame: 13, time sec 1 msec 219 id:281469893186016
Acquired Frame: 14, time sec 1 msec 299 id:281469943542240
Acquired Frame: 14, time sec 1 msec 305 id:281469918364128
Acquired Frame: 14, time sec 1 msec 311 id:281470238724576
Acquired Frame: 14, time sec 1 msec 319 id:281469893186016
Acquired Frame: 15, time sec 1 msec 397 id:281469918364128
Acquired Frame: 15, time sec 1 msec 404 id:281469943542240
Acquired Frame: 15, time sec 1 msec 411 id:281470238724576
Acquired Frame: 15, time sec 1 msec 418 id:281469893186016
Acquired Frame: 16, time sec 1 msec 497 id:281470238724576
Acquired Frame: 16, time sec 1 msec 504 id:281469918364128
Acquired Frame: 16, time sec 1 msec 512 id:281469893186016
Acquired Frame: 16, time sec 1 msec 519 id:281469943542240
Acquired Frame: 17, time sec 1 msec 597 id:281470238724576
Acquired Frame: 17, time sec 1 msec 604 id:281469943542240
Acquired Frame: 17, time sec 1 msec 610 id:281469918364128
Acquired Frame: 17, time sec 1 msec 618 id:281469893186016
Acquired Frame: 18, time sec 1 msec 698 id:281470238724576
Acquired Frame: 18, time sec 1 msec 704 id:281469893186016
Acquired Frame: 18, time sec 1 msec 710 id:281469943542240
Acquired Frame: 18, time sec 1 msec 719 id:281469918364128
Acquired Frame: 19, time sec 1 msec 804 id:281469943542240
Acquired Frame: 19, time sec 1 msec 805 id:281469893186016
Acquired Frame: 19, time sec 1 msec 818 id:281470238724576
Acquired Frame: 19, time sec 1 msec 840 id:281469918364128
Acquired Frame: 20, time sec 1 msec 898 id:281469893186016
Acquired Frame: 20, time sec 1 msec 919 id:281469943542240
Acquired Frame: 20, time sec 1 msec 922 id:281470238724576
Acquired Frame: 20, time sec 1 msec 955 id:281469918364128
Acquired Frame: 21, time sec 1 msec 998 id:281469893186016
Acquired Frame: 21, time sec 2 msec 11 id:281469943542240
Acquired Frame: 22, time sec 2 msec 111 id:281469893186016
Acquired Frame: 22, time sec 2 msec 107 id:281469943542240
Acquired Frame: 22, time sec 2 msec 139 id:281470238724576
Acquired Frame: 22, time sec 2 msec 165 id:281469918364128
Acquired Frame: 23, time sec 2 msec 205 id:281469893186016
Acquired Frame: 23, time sec 2 msec 210 id:28146994354224
Acquired Frame: 23, time sec 2 msec 245 id:281470238724576
Acquired Frame: 23, time sec 2 msec 259 id:281469918364128
Acquired Frame: 24, time sec 2 msec 297 id:281469943542240
Acquired Frame: 24, time sec 2 msec 312 id:281469893186016
Acquired Frame: 24, time sec 2 msec 341 id:281469918364128
Acquired Frame: 24, time sec 2 msec 329 id:281470238724576

The above is part of the log I printed, Frame: 21 has two shots of the data is lost.
The following is the same MP4 file extracted from two consecutive frames of the same picture, but the size is not the same, which is also let me more strange place

hi, DaneLLL
You can help me solve this problem

Hi,
We don’t have much experience in rtmp. Suggest you try UDP or RTSP:
Jetson AGX Orin FAQ

Q: Is there any example of running RTSP streaming?
Q: Is there an example for running UDP streaming?

And the data may be overwritten while using multifilesink. We suggest send the frame data to encoder directly.

All my print information is printed in the nvarguscamerasrc plugin, which means that frames are lost from nvarguscamerasrc. As for the multifilesink problem you mentioned, I divided the fetch stream into three ways, one is nv12 and the other is MP4. One to nvcompositor, you mean that these three paths also affect each other, resulting in data overwriting?

Hi,
In nvarguscamerasrc plugin, there is no mechanism to drop frames. So it may be some processing takes longer time and holds the buffers for next capture. You may add prints in nvarguscamerasrc to check further.

And please run the script and check if it helps:
VPI - Vision Programming Interface: Performance Benchmark
The script enables VIC engine at maximum clock. This enables NvBufSurfTransform() function call to offer optimal throughput. The function is called in nvarguscamerasrc and performance may be better if it offers maximum throughput.

This script has indeed been greatly improved, and the probability of non-synchronization has been greatly improved compared with before, but there is still the phenomenon of non-synchronization. Although there is no frame drop mechanism in nvarguscamerasrc, is it possible that the data is not actually lost but overwritten?

Hi,

This may not be possible. If the buffer is not returned back for capturing next frames quick enough, one frame will be dropped. It will not be overwritten.

Maybe the buffer returned here is too slow,I’ve actually increased MIN_BUFFER and MAX_BUFFER, but it doesn’t seem to have much effect. There are ways to simplify the code and improve performance.

static gpointer
consumer_thread (gpointer src_base)
{
  gint retn = 0;
  GstBuffer *buffer = NULL;
  GstMemory *mem = NULL;
  NvArgusFrameInfo *consumerFrameInfo = NULL;
  GstFlowReturn ret = GST_FLOW_OK;
  static GQuark gst_buffer_metadata_quark = 0;
  gst_buffer_metadata_quark = g_quark_from_static_string ("GstBufferMetaData");

  GstNvArgusCameraSrc *src = (GstNvArgusCameraSrc *) src_base;

  while (FALSE == src->stop_requested)
  {
    g_mutex_lock (&src->argus_buffers_queue_lock);
    if (src->stop_requested || src->timeout_complete)
    {
      g_mutex_unlock (&src->argus_buffers_queue_lock);
      goto done;
    }
    while (g_queue_is_empty (src->argus_buffers) && src->argus_in_error == FALSE)
    {
      gint64 until;
      gboolean ret_val = false;
      until = g_get_monotonic_time () + G_TIME_SPAN_SECOND;
      ret_val = g_cond_wait_until (&src->argus_buffers_queue_cond, &src->argus_buffers_queue_lock, until);
      if (ret_val)
        break;
      else
      {
        if (src->stop_requested || src->timeout_complete)
        {
          g_mutex_unlock(&src->argus_buffers_queue_lock);
          goto done;
        }
      }
    }

    consumerFrameInfo = (NvArgusFrameInfo *) g_queue_pop_head (src->argus_buffers);
    g_mutex_unlock (&src->argus_buffers_queue_lock);
    if(&consumerFrameInfo->fd == NULL)
    {
      goto done;
    }
    ret = gst_buffer_pool_acquire_buffer (src->pool, &buffer, NULL);

    if (ret != GST_FLOW_OK)
    {
      if (!src->stop_requested || !src->timeout_complete)
      {
        GST_ERROR_OBJECT(src, "Error in pool acquire buffer");
      }
      goto done;
    }

    mem = gst_buffer_peek_memory (buffer, 0);
    GstMapInfo outmap = GST_MAP_INFO_INIT;
    if (!mem) {
      GST_ERROR_OBJECT(src, "no memory block");
      goto done;
    }
    gst_buffer_map (buffer, &outmap, GST_MAP_WRITE);
    NvBufSurface*  surf = (NvBufSurface *)outmap.data;

    NvBufSurface *nvbuf_surf = 0;
    retn = NvBufSurfaceFromFd(consumerFrameInfo->fd, (void**)(&nvbuf_surf));
    if (retn != 0) {
      GST_ERROR_OBJECT(src, "NvBufSurfaceFromFd Failed");
      goto done;
    }
    retn = NvBufSurfTransform(nvbuf_surf, surf, &src->transform_params);
    g_mutex_lock (&src->argus_buffer_consumed_lock);
    g_cond_signal (&src->argus_buffer_consumed_cond);
    src->is_argus_buffer_consumed = TRUE;
    g_mutex_unlock (&src->argus_buffer_consumed_lock);
    if (retn != 0) {
      GST_ERROR_OBJECT(src, "NvBufSurfTransform Failed");
      /* TODO: Check if need to set ->stop_requested flag in error condition */
      goto done;
    }

    gst_buffer_unmap (buffer, &outmap);

    g_mutex_lock (&src->nvmm_buffers_queue_lock);
    g_queue_push_tail (src->nvmm_buffers, buffer);
    g_cond_signal (&src->nvmm_buffers_queue_cond);
    g_mutex_unlock (&src->nvmm_buffers_queue_lock);
  }
done:
  GST_DEBUG_OBJECT (src, "%s: stop_requested=%d\n", __func__, src->stop_requested);
  if (buffer)
  {
    GstMiniObject *mobj = NULL;
    mobj = GST_MINI_OBJECT_CAST (buffer);
    if (gst_mini_object_is_writable(mobj))
      gst_mini_object_set_qdata (mobj, gst_buffer_metadata_quark, NULL, NULL);
  }
  return NULL;
}

This is where I actually print the log, followed by the loop waiting buffer

      {
        guint64 frame_timestamp = iFrame->getTime() - ground_clk;
        guint64 millisec_timestamp = ((frame_timestamp % (1000000000)))/1000000;
        printf("Acquired Frame: %llu, time sec %llu msec %llu id:%llu\n",
                   static_cast<unsigned long long>(iFrame->getNumber()),
                   static_cast<unsigned long long>(frame_timestamp / (1000000000)),
                   static_cast<unsigned long long>(millisec_timestamp),threadId);
      }

      src->frameInfo->frameNum = iFrame->getNumber();
      src->frameInfo->frameTime = iFrame->getTime();

      g_mutex_lock (&src->argus_buffers_queue_lock);
      g_queue_push_tail (src->argus_buffers, (src->frameInfo));
      g_cond_signal (&src->argus_buffers_queue_cond);
      g_mutex_unlock (&src->argus_buffers_queue_lock);

      g_mutex_lock (&src->argus_buffer_consumed_lock);
      while (!src->is_argus_buffer_consumed)
      {
        gint64 until;
        until = g_get_monotonic_time () + G_TIME_SPAN_SECOND;
        g_cond_wait_until (&src->argus_buffer_consumed_cond, &src->argus_buffer_consumed_lock, until);
      }
      src->is_argus_buffer_consumed = FALSE;
      g_mutex_unlock (&src->argus_buffer_consumed_lock);
    }

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.