Deadlock in v4l2 encoder abort function

Hi,

i found deadlock in v4l2 h264 encoder nonblocking api. I need to stop encoder, at any time, and the data to enter the encoder does not come immediately. For this abort and usleep. Reproduced if you start several times and stop in the middle using ctrl-c

bt

(gdb) bt
#0  0x0000007fa43bf22c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55a33e3848) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0x55a33e37f0, cond=0x55a33e3820) at pthread_cond_wait.c:502
#2  __pthread_cond_wait (cond=0x55a33e3820, mutex=0x55a33e37f0) at pthread_cond_wait.c:655
#3  0x0000007fa390ffdc in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libnvos.so
#4  0x0000007fa0e6db08 in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libnvmmlite_video.so
#5  0x0000007fa0e27c34 in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libnvmmlite.so
#6  0x0000007fa0eb4e2c in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#7  0x0000007fa0ea76fc in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#8  0x0000007fa0eab810 in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#9  0x0000007fa0e9f960 in TegraV4L2_Ioctl () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#10 0x0000007fa387ae78 in plugin_ioctl () from /usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvvideocodec.so
#11 0x0000007fa42a15c0 in v4l2_ioctl () from /usr/lib/aarch64-linux-gnu/libv4l2.so.0
#12 0x0000005588624f60 in NvV4l2ElementPlane::setStreamStatus(bool) ()
#13 0x000000558862b4d4 in NvV4l2Element::abort() ()
#14 0x00000055885eb5f0 in encoder_proc_nonblocking(context_t&, bool) ()
#15 0x00000055885eec1c in encode_proc(context_t&, int, char**) ()
#16 0x00000055885ef500 in main ()
(gdb) info threads
  Id   Target Id         Frame 
* 1    Thread 0x7fa38af9b0 (LWP 18395) "EncCapPlane" __pthread_cond_wait (cond=0x55a33e3820, mutex=0x55a33e37f0) at pthread_cond_wait.c:655
  2    Thread 0x7fa34ea1d0 (LWP 18396) "video_encode" 0x0000007fa43bf22c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55a338b788)
    at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  3    Thread 0x7fa2ce91d0 (LWP 18397) "video_encode" 0x0000007fa43bf22c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55a338b758)
    at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  4    Thread 0x7f9e83f1d0 (LWP 18398) "V4L2_EncThread" 0x0000007fa43bf22c in futex_wait_cancelable (private=<optimized out>, expected=0, 
    futex_word=0x55a33a830c) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  5    Thread 0x7f9e03e1d0 (LWP 18399) "EncPollThread" 0x0000007fa43c19c8 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, 
    futex_word=0x558864c2b8 <ctx+568>) at ../sysdeps/unix/sysv/linux/futex-internal.h:205
  6    Thread 0x7f97fff1d0 (LWP 18400) "NVMVidEncInputT" 0x0000007fa43bf22c in futex_wait_cancelable (private=<optimized out>, expected=0, 
    futex_word=0x55a33e38bc) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  7    Thread 0x7f977fe1d0 (LWP 18401) "V4L2_EncThread" 0x0000007fa43bf22c in futex_wait_cancelable (private=<optimized out>, expected=0, 
    futex_word=0x55a33e27bc) at ../sysdeps/unix/sysv/linux/futex-internal.h:88

part
info proc mapping

        0x7fa0e5b000       0x7fa0e8a000    0x2f000        0x0 /usr/lib/aarch64-linux-gnu/tegra/libnvmmlite_video.so
        0x7fa0e8a000       0x7fa0e9a000    0x10000    0x2f000 /usr/lib/aarch64-linux-gnu/tegra/libnvmmlite_video.so
        0x7fa0e9a000       0x7fa0e9b000     0x1000    0x2f000 /usr/lib/aarch64-linux-gnu/tegra/libnvmmlite_video.so
        0x7fa0e9b000       0x7fa0e9d000     0x2000    0x30000 /usr/lib/aarch64-linux-gnu/tegra/libnvmmlite_video.so
        0x7fa0e9d000       0x7fa0ec9000    0x2c000        0x0 /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
        0x7fa0ec9000       0x7fa0ed8000     0xf000    0x2c000 /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
        0x7fa0ed8000       0x7fa0ed9000     0x1000    0x2b000 /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
        0x7fa0ed9000       0x7fa0eda000     0x1000    0x2c000 /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so

jetson tx2 8Gb
l4t # R32 (release), REVISION: 4.3, GCID: 21589087, BOARD: t186ref, EABI: aarch64, DATE: Fri Jun 26 04:34:27 UTC 2020

./video_encode source.yuv 1280 720 H264 dst.h264 --blocking-mode 0

Patch video_encode sample

# diff -ruN samples/01_video_encode/video_encode_main.cpp.orig samples/01_video_encode/video_encode_main.cpp 
--- samples/01_video_encode/video_encode_main.cpp.orig	2020-07-15 17:28:23.281784416 +0300
+++ samples/01_video_encode/video_encode_main.cpp	2020-07-15 18:24:35.215037494 +0300
@@ -35,6 +35,8 @@
 #include <string.h>
 #include <fcntl.h>
 #include <poll.h>
+#include <unistd.h>
+#include <signal.h>
 #include "nvbuf_utils.h"
 
 #include "video_encode.h"
@@ -547,6 +549,7 @@
     ctx->endf = 0;
     ctx->num_output_buffers = 6;
     ctx->num_frames_to_encode = -1;
+    ctx->stop = false;
 }
 
 /**
@@ -814,12 +817,21 @@
     {
         /* Call SetPollInterrupt */
         ctx.enc->SetPollInterrupt();
+        if (ctx.stop) {
+            ctx.enc->abort();
+            break;
+        }
 
         /* Since buffers have been queued, issue a post to start polling and
            then wait here */
         sem_post(&ctx.pollthread_sema);
         sem_wait(&ctx.encoderthread_sema);
 
+        if (ctx.stop) {
+            ctx.enc->abort();
+            break;
+        }
+
         /* Already end of file, no more queue-dequeue for output plane */
         if (eos)
             goto check_capture_buffers;
@@ -849,6 +861,11 @@
                 abort(&ctx);
                 return -1;
             }
+            if (ctx.stop) {
+                ctx.enc->abort();
+                break;
+            }
+
 
             /* Get the parsed encoder runtime parameters */
             if (ctx.runtime_params_str &&
@@ -859,6 +876,12 @@
                 if (ctx.runtime_params_str)
                     get_next_runtime_param_change_frame(&ctx);
             }
+            usleep(500);
+
+            if (ctx.stop) {
+                ctx.enc->abort();
+                break;
+            }
 
             /* Read yuv frame data from input file */
             if (read_video_frame(ctx.in_file, *outplane_buffer) < 0 || ctx.num_frames_to_encode == 0)
@@ -1006,6 +1029,11 @@
                 abort(&ctx);
                 return -1;
             }
+            if (ctx.stop) {
+                ctx.enc->abort();
+                break;
+            }
+
             if(ctx.num_frames_to_encode > 0)
             {
                 ctx.num_frames_to_encode--;
@@ -1043,6 +1071,11 @@
                 abort(&ctx);
                 return -1;
             }
+            if (ctx.stop) {
+                ctx.enc->abort();
+                break;
+            }
+
 
             /* Invoke encoder capture-plane deque buffer callback */
             capture_dq_continue = encoder_capture_plane_dq_callback(&v4l2_capture_buf, capplane_buffer, NULL,
@@ -1053,6 +1086,11 @@
                 ctx.got_eos = true;
                 return 0;
             }
+            if (ctx.stop) {
+                ctx.enc->abort();
+                break;
+            }
+
         }
     }
 
@@ -2071,6 +2109,18 @@
     return -error;
 }
 
+/* create encoder context. */
+context_t ctx;
+
+void hdnl(int sig) {
+    cerr << "siganal " << sig << endl;
+    if (ctx.enc) {
+        cerr << "abort " << endl;
+        ctx.stop = true;
+    }
+}
+
+
 /**
   * Start of video Encode application.
   *
@@ -2080,11 +2130,10 @@
 int
 main(int argc, char *argv[])
 {
-    /* create encoder context. */
-    context_t ctx;
     int ret = 0;
     /* save encode iterator number */
     int iterator_num = 0;
+    signal(SIGINT, &hdnl);
 
     do
     {
--- samples/01_video_encode/video_encode.h.orig 2020-07-15 18:41:31.816189379 +0300
+++ samples/01_video_encode/video_encode.h      2020-07-15 17:56:28.885867902 +0300
@@ -158,6 +158,7 @@
     sem_t encoderthread_sema; // Encoder thread waits on this to be signalled to continue q/dq loop
     pthread_t   enc_pollthread; // Polling thread, created if running in non-blocking mode.
     pthread_t enc_capture_loop; // Encoder capture thread
+    bool stop;
 } context_t;

Hi,
For this case, we would suggest issue stop command or send EoS to output plane. Please refer to the code about num_frames_to_encode:

            v4l2_buf.m.planes[0].bytesused = 0;
            if(ctx.b_use_enc_cmd)
             {
                /* Send v4l2 command for encoder stop */
                ret = ctx.enc->setEncoderCommand(V4L2_ENC_CMD_STOP, 1);
                eos = true;
                break;
             }
             else
             {
                 eos = true;
                 v4l2_buf.m.planes[0].m.userptr = 0;
                 v4l2_buf.m.planes[0].bytesused = v4l2_buf.m.planes[1].bytesused = v4l2_buf.m.planes[2].bytesused = 0;
             }

I also checked V4L2_ENC_CMD_STOP

when Flag set 1, also deadlock in abort, when Flag set 0 deadlock in setEncoderCommand

Flag 1

(gdb) bt
#0  0x0000007f89d4e22c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x558fc8f848)
    at ../sysdeps/unix/sysv/linux/futex-internal.h:88
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0x558fc8f7f0, cond=0x558fc8f820) at pthread_cond_wait.c:502
#2  __pthread_cond_wait (cond=0x558fc8f820, mutex=0x558fc8f7f0) at pthread_cond_wait.c:655
#3  0x0000007f8929efdc in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libnvos.so
#4  0x0000007f867fcb08 in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libnvmmlite_video.so
#5  0x0000007f867b6c34 in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libnvmmlite.so
#6  0x0000007f86843e2c in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#7  0x0000007f868366fc in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#8  0x0000007f8683a810 in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#9  0x0000007f8682e960 in TegraV4L2_Ioctl () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#10 0x0000007f89209e78 in plugin_ioctl () from /usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvvideocodec.so
#11 0x0000007f89c305c0 in v4l2_ioctl () from /usr/lib/aarch64-linux-gnu/libv4l2.so.0
#12 0x0000005588747fa8 in NvV4l2ElementPlane::setStreamStatus (this=0x558fc475d8, status=false) at NvV4l2ElementPlane.cpp:546
#13 0x000000558874d51c in NvV4l2Element::abort (this=0x558fc474d0) at NvV4l2Element.cpp:244
#14 0x000000558870e400 in abrt (ctx=...) at video_encode_main.cpp:800
#15 0x000000558870ebac in encoder_proc_nonblocking (ctx=..., eos=false) at video_encode_main.cpp:1040
#16 0x0000005588711c64 in encode_proc (ctx=..., argc=8, argv=0x7fd603d308) at video_encode_main.cpp:2012
#17 0x0000005588712548 in main (argc=8, argv=0x7fd603d308) at video_encode_main.cpp:2148
(gdb) frame 14
#14 0x000000558870e400 in abrt (ctx=...) at video_encode_main.cpp:800
800         ctx.enc->abort();
(gdb) list
795
796     void abrt(context_t &ctx)
797     {
798         cerr << "abrt" << endl;
799         ctx.enc->setEncoderCommand(V4L2_ENC_CMD_STOP, 1);
800         ctx.enc->abort();
801     }
802
803     /**
804       * Encode processing function for non-blocking mode.

Flag 0

(gdb) bt
#0  0x0000007fad61622c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55b25d0848)
    at ../sysdeps/unix/sysv/linux/futex-internal.h:88
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0x55b25d07f0, cond=0x55b25d0820) at pthread_cond_wait.c:502
#2  __pthread_cond_wait (cond=0x55b25d0820, mutex=0x55b25d07f0) at pthread_cond_wait.c:655
#3  0x0000007facb66fdc in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libnvos.so
#4  0x0000007faa0c4b08 in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libnvmmlite_video.so
#5  0x0000007faa07ec34 in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libnvmmlite.so
#6  0x0000007faa10be2c in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#7  0x0000007faa0fe6fc in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#8  0x0000007faa100520 in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#9  0x0000007faa1013e0 in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#10 0x0000007faa0f6960 in TegraV4L2_Ioctl () from /usr/lib/aarch64-linux-gnu/tegra/libtegrav4l2.so
#11 0x0000007facad1e78 in plugin_ioctl () from /usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvvideocodec.so
#12 0x0000007fad4f85c0 in v4l2_ioctl () from /usr/lib/aarch64-linux-gnu/libv4l2.so.0
#13 0x000000557da52028 in NvVideoEncoder::setEncoderCommand (this=0x55b25884d0, cmd=1, flags=0) at NvVideoEncoder.cpp:202
#14 0x000000557da2d3f4 in abrt (ctx=...) at video_encode_main.cpp:799
#15 0x000000557da2dbac in encoder_proc_nonblocking (ctx=..., eos=false) at video_encode_main.cpp:1040
#16 0x000000557da30c64 in encode_proc (ctx=..., argc=8, argv=0x7fde2f4098) at video_encode_main.cpp:2012
#17 0x000000557da31548 in main (argc=8, argv=0x7fde2f4098) at video_encode_main.cpp:2148
(gdb) list
83      in ../sysdeps/unix/sysv/linux/futex-internal.h
(gdb) frame 14
#14 0x000000557da2d3f4 in abrt (ctx=...) at video_encode_main.cpp:799
799         ctx.enc->setEncoderCommand(V4L2_ENC_CMD_STOP, 0);
(gdb) list
794     }
795
796     void abrt(context_t &ctx)
797     {
798         cerr << "abrt" << endl;
799         ctx.enc->setEncoderCommand(V4L2_ENC_CMD_STOP, 0);
800         ctx.enc->abort();
801     }
802
803     /**

I would like not to use an empty buffer as eos, because most often, all buffers are busy and you need to wait for the free one.

0x0000007faa0c4b08 in ?? () from /usr/lib/aarch64-linux-gnu/tegra/libnvmmlite_video.so

Assembler code in libnvmmlite_video.so 0x0000007faa0c4b08 looks like code that stop encoder unless it first gets into a deadlock with some code somewhere else.

Hi,
After issuing stop command, you would need to wait for all buffers in output plane and capture plane being sent back from encoder to application layer, and then terminate encoder.

Hi,

does not help , also deadlock in the same place. Also, with existing /tmp/tegrav4l2_logs file i see additional debug logs. So, when happy path she looks like

^Csiganal 2                                                                                                                          [1760/740897]
abort                                                                                       
abrt                                                                                        
(tid): 787939b0 LIBTEGRAV4L2: ENC_CTX(0x55a74240e0) Current State V4L2_ENCODER_STATE_RUNNING & Received CMD V4L2_ENC_CMD_STOP
(tid): 787939b0 LIBTEGRAV4L2:                                                               
Stop using V4L2_ENC_CMD_STOP_AT_GOP_END                                                     
(tid): 737231d0 LIBTEGRAV4L2: v4l2_enc_nvmmlite_allocate_and_send_eos_output_buffer: Sending OUTPUT_PLANE buffer of size 0
(tid): 737231d0 LIBTEGRAV4L2: Function v4l2_enc_nvmmlite_allocate_and_send_eos_output_buffer: Sent event of EOS
(tid): 70f531d0 LIBTEGRAV4L2: NvVideoEnc: Encoded Frame Num: 226, Size: 43810 bytes (350480 bits)
(tid): 787939b0 LIBTEGRAV4L2: ENC_CTX(0x55a74240e0) Output stream OFF                       
(tid): 70f531d0 LIBTEGRAV4L2: NvVideoEnc: Encoded Frame Num: 227, Size: 44811 bytes (358488 bits)
(tid): 787939b0 LIBTEGRAV4L2: BufferIndex already done 1                                    
(tid): 787939b0 LIBTEGRAV4L2: BufferIndex already done 2                                         
(tid): 787939b0 LIBTEGRAV4L2: BufferIndex already done 3                                    
(tid): 787939b0 LIBTEGRAV4L2: ENC_CTX(0x55a74240e0) Capture stream OFF                           
(tid): 70f531d0 LIBTEGRAV4L2: NvVideoEnc: Encoded Frame Num: 228, Size: 38200 bytes (305600 bits)
(tid): 787939b0 LIBTEGRAV4L2: NvVideoEncTransferBufferFromBlock: Received unused output buffer 2 
(tid): 787939b0 LIBTEGRAV4L2:                                                                    
v4l2_thread_shutdown:Wait on join to thread tegrav4l2_enc_thread_func                            
(tid): 737231d0 LIBTEGRAV4L2: ENC_CTX(0x55a74240e0) Exiting from tegrav4l2 thread                

deadlock looks like:

^Csiganal 2                                                                                      
abort                                                                                                       
abrt                                                                                             
(tid): 9f3db9b0 LIBTEGRAV4L2: ENC_CTX(0x5562d9d0e0) Current State V4L2_ENCODER_STATE_RUNNING & Received CMD V4L2_ENC_CMD_STOP
(tid): 9f3db9b0 LIBTEGRAV4L2:                                                                    
Stop using V4L2_ENC_CMD_STOP_AT_GOP_END                                                          
(tid): 9a36b1d0 LIBTEGRAV4L2: v4l2_enc_nvmmlite_allocate_and_send_eos_output_buffer: Sending OUTPUT_PLANE buffer of size 0                        
(tid): 9a36b1d0 LIBTEGRAV4L2: Function v4l2_enc_nvmmlite_allocate_and_send_eos_output_buffer: Sent event of EOS
(tid): 9f3db9b0 LIBTEGRAV4L2: ENC_CTX(0x5562d9d0e0) Output stream OFF                               

my function with send stop and waiting for as many buffers as possible

void abrt(context_t &ctx)
{
    cerr << "abrt" << endl;
    ctx.enc->setEncoderCommand(V4L2_ENC_CMD_STOP, 1);
    int ret;
    
    struct v4l2_buffer v4l2_output_buf;
    struct v4l2_plane output_planes[MAX_PLANES];
    NvBuffer *outplane_buffer = NULL;

    struct v4l2_buffer v4l2_capture_buf;
    struct v4l2_plane capture_planes[MAX_PLANES];
    NvBuffer *capplane_buffer = NULL;
    int count = 0;

    do {
        do {
            memset(&v4l2_output_buf, 0, sizeof(v4l2_output_buf));
            memset(output_planes, 0, sizeof(output_planes));
            v4l2_output_buf.m.planes = output_planes;
            ret = ctx.enc->output_plane.dqBuffer(v4l2_output_buf, &outplane_buffer, NULL, 10);
        } while (ret == 0);

        do {
            memset(&v4l2_capture_buf, 0, sizeof(v4l2_capture_buf));
            memset(capture_planes, 0, sizeof(capture_planes));
            v4l2_capture_buf.m.planes = capture_planes;
            v4l2_capture_buf.length = 1;

            /* Dequeue from output plane, fill the frame and enqueue it back again.
                NOTE: This could be moved out to a different thread as an optimization. */
            ret = ctx.enc->capture_plane.dqBuffer(v4l2_capture_buf, &capplane_buffer, NULL, 10);
        } while (ret == 0);
        count++;
        usleep(100);
    } while (count < 10);

    ctx.enc->abort();
}

Hi,
Please refer to the patch:

diff --git a/multimedia_api/ll_samples/samples/01_video_encode/video_encode_main.cpp b/multimedia_api/ll_samples/samples/01_video_encode/video_encode_main.cpp
index d68d4ec..60b4727 100644
--- a/multimedia_api/ll_samples/samples/01_video_encode/video_encode_main.cpp
+++ b/multimedia_api/ll_samples/samples/01_video_encode/video_encode_main.cpp
@@ -39,6 +39,8 @@
 
 #include "video_encode.h"
 
+#include <signal.h>
+
 #define TEST_ERROR(cond, str, label) if(cond) { \
                                         cerr << str << endl; \
                                         error = 1; \
@@ -53,6 +55,12 @@
 
 using namespace std;
 
+static int signal_stop = 0;
+void hdnl(int sig) {
+    cerr << "siganal " << sig << endl;
+    signal_stop = 1;
+}
+
 /**
   * Abort on error.
   *
@@ -861,7 +869,8 @@ static int encoder_proc_nonblocking(context_t &ctx, bool eos)
             }
 
             /* Read yuv frame data from input file */
-            if (read_video_frame(ctx.in_file, *outplane_buffer) < 0 || ctx.num_frames_to_encode == 0)
+            if (signal_stop ||
+                read_video_frame(ctx.in_file, *outplane_buffer) < 0 || ctx.num_frames_to_encode == 0)
             {
                 cerr << "Could not read complete frame from input file" << endl;
                 v4l2_output_buf.m.planes[0].bytesused = 0;
@@ -2086,6 +2095,8 @@ main(int argc, char *argv[])
     /* save encode iterator number */
     int iterator_num = 0;
 
+    signal(SIGINT, &hdnl);
+
     do
     {
         /* Invoke video encode function. */

You don’t need to call abort() in normal exit.

The command for verification is

$ ./video_encode a.yuv 3840 2160 H264 a.264 --blocking-mode 0 --enc-cmd

Oh, i see, stop called from enc destructor. Your patch stop only after output_plane.dqBuffer but i like stop in anywhere in function encoder_proc_nonblocking.

My clean function also deadlock, also ctx.enc->output_plane.getNumQueuedBuffers() == 0 never happened

    do {
        ctx.enc->SetPollInterrupt();
        sem_post(&ctx.pollthread_sema);
        sem_wait(&ctx.encoderthread_sema);


        do {
            memset(&v4l2_output_buf, 0, sizeof(v4l2_output_buf));
            memset(output_planes, 0, sizeof(output_planes));
            v4l2_output_buf.m.planes = output_planes;
            ret = ctx.enc->output_plane.dqBuffer(v4l2_output_buf, &outplane_buffer, NULL, 10);
            if (ret == 0) cerr << "output_plane.dqBuffer " << ret << " " << outplane_buffer << endl;
        } while (ret == 0);

        do {
            memset(&v4l2_capture_buf, 0, sizeof(v4l2_capture_buf));
            memset(capture_planes, 0, sizeof(capture_planes));
            v4l2_capture_buf.m.planes = capture_planes;
            v4l2_capture_buf.length = 1;

            ret = ctx.enc->capture_plane.dqBuffer(v4l2_capture_buf, &capplane_buffer, NULL, 10);
            if (ret == 0) cerr << "capture_plane.dqBuffer " << ret << " " << capplane_buffer << endl;
        } while (ret == 0); 
        cerr << "output_plane queued buffers " << ctx.enc->output_plane.getNumQueuedBuffers() << endl;
        cerr << "capture_plane queued buffers " << ctx.enc->capture_plane.getNumQueuedBuffers() << endl;
    } while (ctx.enc->capture_plane.getNumQueuedBuffers());
--- video_encode_main.cpp.orig	2020-07-15 17:28:23.281784416 +0300
+++ video_encode_main.cpp	2020-07-17 10:41:17.509259363 +0300
@@ -35,6 +35,8 @@
 #include <string.h>
 #include <fcntl.h>
 #include <poll.h>
+#include <unistd.h>
+#include <signal.h>
 #include "nvbuf_utils.h"
 
 #include "video_encode.h"
@@ -547,6 +549,8 @@
     ctx->endf = 0;
     ctx->num_output_buffers = 6;
     ctx->num_frames_to_encode = -1;
+    ctx->stop = false;
+    ctx->dq_v4l2_output_buf = false;
 }
 
 /**
@@ -790,6 +794,7 @@
     return NULL;
 }
 
+
 /**
   * Encode processing function for non-blocking mode.
   *
@@ -814,12 +819,19 @@
     {
         /* Call SetPollInterrupt */
         ctx.enc->SetPollInterrupt();
+        if (ctx.stop) {
+            break;
+        }
 
         /* Since buffers have been queued, issue a post to start polling and
            then wait here */
         sem_post(&ctx.pollthread_sema);
         sem_wait(&ctx.encoderthread_sema);
 
+        if (ctx.stop) {
+            break;
+        }
+
         /* Already end of file, no more queue-dequeue for output plane */
         if (eos)
             goto check_capture_buffers;
@@ -827,18 +839,16 @@
         /* Check if can dequeue from output plane */
         while (1)
         {
-
-            struct v4l2_buffer v4l2_output_buf;
             struct v4l2_plane output_planes[MAX_PLANES];
             NvBuffer *outplane_buffer = NULL;
 
-            memset(&v4l2_output_buf, 0, sizeof(v4l2_output_buf));
+            memset(&ctx.v4l2_output_buf, 0, sizeof(ctx.v4l2_output_buf));
             memset(output_planes, 0, sizeof(output_planes));
-            v4l2_output_buf.m.planes = output_planes;
+            ctx.v4l2_output_buf.m.planes = output_planes;
 
             /* Dequeue from output plane, fill the frame and enqueue it back again.
                NOTE: This could be moved out to a different thread as an optimization. */
-            ret = ctx.enc->output_plane.dqBuffer(v4l2_output_buf, &outplane_buffer, NULL, 10);
+            ret = ctx.enc->output_plane.dqBuffer(ctx.v4l2_output_buf, &outplane_buffer, NULL, 10);
             if (ret < 0)
             {
                 if (errno == EAGAIN)
@@ -849,6 +859,11 @@
                 abort(&ctx);
                 return -1;
             }
+            ctx.dq_v4l2_output_buf = true;
+            if (ctx.stop) {
+                break;
+            }
+
 
             /* Get the parsed encoder runtime parameters */
             if (ctx.runtime_params_str &&
@@ -859,12 +874,17 @@
                 if (ctx.runtime_params_str)
                     get_next_runtime_param_change_frame(&ctx);
             }
+            usleep(500);
+
+            if (ctx.stop) {
+                break;
+            }
 
             /* Read yuv frame data from input file */
             if (read_video_frame(ctx.in_file, *outplane_buffer) < 0 || ctx.num_frames_to_encode == 0)
             {
                 cerr << "Could not read complete frame from input file" << endl;
-                v4l2_output_buf.m.planes[0].bytesused = 0;
+                ctx.v4l2_output_buf.m.planes[0].bytesused = 0;
                 if(ctx.b_use_enc_cmd)
                 {
                     ret = ctx.enc->setEncoderCommand(V4L2_ENC_CMD_STOP, 1);
@@ -874,10 +894,10 @@
                 else
                 {
                     eos = true;
-                    v4l2_output_buf.m.planes[0].m.userptr = 0;
-                    v4l2_output_buf.m.planes[0].bytesused = 0;
-                    v4l2_output_buf.m.planes[1].bytesused = 0;
-                    v4l2_output_buf.m.planes[2].bytesused = 0;
+                    ctx.v4l2_output_buf.m.planes[0].m.userptr = 0;
+                    ctx.v4l2_output_buf.m.planes[0].bytesused = 0;
+                    ctx.v4l2_output_buf.m.planes[1].bytesused = 0;
+                    ctx.v4l2_output_buf.m.planes[2].bytesused = 0;
                 }
             }
 
@@ -958,18 +978,18 @@
                 if (VEnc_imeta_param.flag)
                 {
                     /* Set encoder input metadatas */
-                    ctx.enc->SetInputMetaParams(v4l2_output_buf.index, VEnc_imeta_param);
-                    v4l2_output_buf.reserved2 = v4l2_output_buf.index;
+                    ctx.enc->SetInputMetaParams(ctx.v4l2_output_buf.index, VEnc_imeta_param);
+                    ctx.v4l2_output_buf.reserved2 = ctx.v4l2_output_buf.index;
                 }
             }
 
             if (ctx.copy_timestamp)
             {
                 /* Set user provided timestamp when copy timestamp is enabled */
-                v4l2_output_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY;
+                ctx.v4l2_output_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY;
                 ctx.timestamp += ctx.timestampincr;
-                v4l2_output_buf.timestamp.tv_sec = ctx.timestamp / (MICROSECOND_UNIT);
-                v4l2_output_buf.timestamp.tv_usec = ctx.timestamp % (MICROSECOND_UNIT);
+                ctx.v4l2_output_buf.timestamp.tv_sec = ctx.timestamp / (MICROSECOND_UNIT);
+                ctx.v4l2_output_buf.timestamp.tv_usec = ctx.timestamp % (MICROSECOND_UNIT);
             }
 
             if(!ctx.num_frames_to_encode)
@@ -995,23 +1015,28 @@
             {
                 for (uint32_t j = 0 ; j < outplane_buffer->n_planes; j++)
                 {
-                    v4l2_output_buf.m.planes[j].bytesused = outplane_buffer->planes[j].bytesused;
+                    ctx.v4l2_output_buf.m.planes[j].bytesused = outplane_buffer->planes[j].bytesused;
                 }
             }
             /* encoder qbuffer for output plane */
-            ret = ctx.enc->output_plane.qBuffer(v4l2_output_buf, NULL);
+            ret = ctx.enc->output_plane.qBuffer(ctx.v4l2_output_buf, NULL);
             if (ret < 0)
             {
                 cerr << "Error while queueing buffer at output plane" << endl;
                 abort(&ctx);
                 return -1;
             }
+            ctx.dq_v4l2_output_buf = false;
+            if (ctx.stop) {
+                break;
+            }
+
             if(ctx.num_frames_to_encode > 0)
             {
                 ctx.num_frames_to_encode--;
             }
             ctx.input_frames_queued_count++;
-            if (v4l2_output_buf.m.planes[0].bytesused == 0)
+            if (ctx.v4l2_output_buf.m.planes[0].bytesused == 0)
             {
                 cerr << "File read complete." << endl;
                 eos = true;
@@ -1043,6 +1068,10 @@
                 abort(&ctx);
                 return -1;
             }
+            if (ctx.stop) {
+                break;
+            }
+
 
             /* Invoke encoder capture-plane deque buffer callback */
             capture_dq_continue = encoder_capture_plane_dq_callback(&v4l2_capture_buf, capplane_buffer, NULL,
@@ -1053,6 +1082,10 @@
                 ctx.got_eos = true;
                 return 0;
             }
+            if (ctx.stop) {
+                break;
+            }
+
         }
     }
 
@@ -1984,6 +2017,95 @@
         error = 1;
     }
 
+    struct v4l2_plane output_planes[MAX_PLANES];
+    NvBuffer *outplane_buffer = NULL;
+    struct v4l2_buffer v4l2_capture_buf;
+    struct v4l2_plane capture_planes[MAX_PLANES];
+    NvBuffer *capplane_buffer = NULL;
+
+    //ctx.enc->ClearPollInterrupt();
+
+
+
+
+    if (ctx.dq_v4l2_output_buf) {
+        cerr << "Cleanup Send EOS " << endl;
+        ctx.v4l2_output_buf.m.planes[0].bytesused = 0;
+        /* encoder qbuffer for output plane */
+        ret = ctx.enc->output_plane.qBuffer(ctx.v4l2_output_buf, NULL);
+        if (ret < 0)
+        {
+            cerr << "Cleanup Error while queueing buffer at output plane" << endl;
+            abort(&ctx);
+            return -1;
+        }
+        ctx.dq_v4l2_output_buf = false;
+    } else {
+        cerr << "dqBuffer for EOS " << endl;
+        while (true) {
+            ctx.enc->SetPollInterrupt();
+            sem_post(&ctx.pollthread_sema);
+            sem_wait(&ctx.encoderthread_sema);
+
+            memset(&ctx.v4l2_output_buf, 0, sizeof(ctx.v4l2_output_buf));
+            memset(output_planes, 0, sizeof(output_planes));
+            ctx.v4l2_output_buf.m.planes = output_planes;
+            ret = ctx.enc->output_plane.dqBuffer(ctx.v4l2_output_buf, &outplane_buffer, NULL, 10);
+            if (ret < 0)
+            {
+                cerr << "Cleanup output dqbuffer error, try clean capture plane" << endl;
+                goto clean_capture;
+            }
+            ctx.v4l2_output_buf.m.planes[0].bytesused = 0;
+            cerr << "Cleanup Send EOS " << endl;
+            ret = ctx.enc->output_plane.qBuffer(ctx.v4l2_output_buf, NULL);
+            if (ret < 0)
+            {
+                cerr << "Cleanup Error while queueing buffer at output plane" << endl;                
+                return -1;
+            }
+            break;
+clean_capture:
+            memset(&v4l2_capture_buf, 0, sizeof(v4l2_capture_buf));
+            memset(capture_planes, 0, sizeof(capture_planes));
+            v4l2_capture_buf.m.planes = capture_planes;
+            v4l2_capture_buf.length = 1;
+
+            ctx.enc->capture_plane.dqBuffer(v4l2_capture_buf, &capplane_buffer, NULL, 10);
+        }
+        ctx.dq_v4l2_output_buf = false;
+    }
+    
+    struct v4l2_buffer v4l2_output_buf;
+
+    do {
+        ctx.enc->SetPollInterrupt();
+        sem_post(&ctx.pollthread_sema);
+        sem_wait(&ctx.encoderthread_sema);
+
+
+        do {
+            memset(&v4l2_output_buf, 0, sizeof(v4l2_output_buf));
+            memset(output_planes, 0, sizeof(output_planes));
+            v4l2_output_buf.m.planes = output_planes;
+            ret = ctx.enc->output_plane.dqBuffer(v4l2_output_buf, &outplane_buffer, NULL, 10);
+            if (ret == 0) cerr << "output_plane.dqBuffer " << ret << " " << outplane_buffer << endl;
+        } while (ret == 0);
+
+        do {
+            memset(&v4l2_capture_buf, 0, sizeof(v4l2_capture_buf));
+            memset(capture_planes, 0, sizeof(capture_planes));
+            v4l2_capture_buf.m.planes = capture_planes;
+            v4l2_capture_buf.length = 1;
+
+            ret = ctx.enc->capture_plane.dqBuffer(v4l2_capture_buf, &capplane_buffer, NULL, 10);
+            if (ret == 0) cerr << "capture_plane.dqBuffer " << ret << " " << capplane_buffer << endl;
+        } while (ret == 0); 
+        cerr << "output_plane queued buffers " << ctx.enc->output_plane.getNumQueuedBuffers() << endl;
+        cerr << "capture_plane queued buffers " << ctx.enc->capture_plane.getNumQueuedBuffers() << endl;
+    } while (ctx.enc->capture_plane.getNumQueuedBuffers());
+
+
     if (ctx.pBitStreamCrc)
     {
         char *pgold_crc = ctx.gold_crc;
@@ -2042,6 +2164,8 @@
         }
     }
 
+
+
     /* Release encoder configuration specific resources. */
     delete ctx.enc;
     delete ctx.in_file;
@@ -2071,6 +2195,18 @@
     return -error;
 }
 
+/* create encoder context. */
+context_t ctx;
+
+void hdnl(int sig) {
+    cerr << "siganal " << sig << endl;
+    if (ctx.enc) {
+        cerr << "stop " << endl;
+        ctx.stop = true;
+    }
+}
+
+
 /**
   * Start of video Encode application.
   *
@@ -2080,11 +2216,10 @@
 int
 main(int argc, char *argv[])
 {
-    /* create encoder context. */
-    context_t ctx;
     int ret = 0;
     /* save encode iterator number */
     int iterator_num = 0;
+    signal(SIGINT, &hdnl);
 
     do
     {
@@ -2093,6 +2228,8 @@
         iterator_num++;
     } while((ctx.stress_test != iterator_num) && ret == 0);
 
+
+
     /* Report application run status on exit. */
     if (ret)
     {
--- video_encode.h.orig	2020-07-15 18:41:31.816189379 +0300
+++ video_encode.h	2020-07-17 10:03:13.509555736 +0300
@@ -158,6 +158,9 @@
     sem_t encoderthread_sema; // Encoder thread waits on this to be signalled to continue q/dq loop
     pthread_t   enc_pollthread; // Polling thread, created if running in non-blocking mode.
     pthread_t enc_capture_loop; // Encoder capture thread
+    bool stop;
+    bool dq_v4l2_output_buf;
+    struct v4l2_buffer v4l2_output_buf;
 } context_t;
 
 int parse_csv_args(context_t * ctx, int argc, char *argv[]);

I think this is a bug, non-blocking ioctl should not block randomly.

1 Like

hi,
We think the current implementation is fine and should run well in general usecases. Since the code is open source, you can look at

/usr/src/jetson_multimedia_api/samples/common/classes/NvV4l2Element.cpp
/usr/src/jetson_multimedia_api/samples/common/classes/NvV4l2ElementPlane.cpp
/usr/src/jetson_multimedia_api/samples/common/classes/NvVideoEncoder.cpp

and try to customize it into your usecase.

Hi,

i would not like to use your multimedia api, because c++. Can I use v4l2 api and expect v4l2 api semantics from it? For example, calling v4l2 ioctl in non-blocking mode will never block. It important for me, because i using erlang as a high level language and block inside v4l2 iocl call stop all erlang vm.

Also, I can and do create a separate thread for v4l2 calls, but because of the blocking call, the destructor mechanism in erlang may not work well. Since this is an erlang vm, the process can terminate at any time, and that’s okay.