Multimedia API capture with multi-planer color space

Hi, I am developing encoding program using combination of
03_video_cuda_enc
12_camera_v4l2_cuda

My problem is that 12_camera_v4l2_cuda have only case of
YUV422 interlaced single-planar, but I want to capture by NV12/YU12/YV12 color format witch is multi-planar format.

In sample code 12_camera_v4l2_cuda is
/* TODO: add multi-planar support
Currently only supports YUV422 interlaced single-planar */

Are there some information doing multi-planar format capturing?

Hi,
We support GREY format in 12_camera_v4l2_cuda. GREY is actually NV12/YU12/YV12 without UV plane. You may check how it works, and replace NvBufferColorFormat_GRAY8 with desired color format for a try.

You would need to consider data alignment in NvBuffer. Please refer to this post:
Memory for NvMap - #10 by DaneLLL

Thank you for kind replay!
I will try watching for GREY color format part.

What I want to do is getting YUV420 multi plannar dmabuff_fd of the Camera buffer, but my v4l2_buf always returns single plane at start_capture.

while (poll(fds, 1, 5000) > 0 && !quit)
{
    if (fds[0].revents & POLLIN) {
        struct v4l2_buffer v4l2_buf;

    /* Dequeue a camera buff */
    memset(&v4l2_buf, 0, sizeof(v4l2_buf));
    v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    if (ctx->capture_dmabuf)
        v4l2_buf.memory = V4L2_MEMORY_DMABUF;
    else
        v4l2_buf.memory = V4L2_MEMORY_MMAP;
    if (ioctl(ctx->cam_fd, VIDIOC_DQBUF, &v4l2_buf) < 0)
        ERROR_RETURN("Failed to dequeue camera buff: %s (%d)",
                strerror(errno), errno);

    ctx->frame++;

    /* Save the n-th frame to file */
    if (ctx->frame == ctx->save_n_frame)
        save_frame_to_file(ctx, &v4l2_buf);

    if (ctx->cam_pixfmt == V4L2_PIX_FMT_MJPEG) {
        int fd = 0;
        uint32_t width, height, pixfmt;
        unsigned int i = 0;
        unsigned int eos_search_size = MJPEG_EOS_SEARCH_SIZE;
        unsigned int bytesused = v4l2_buf.bytesused;
        uint8_t *p;

        /* v4l2_buf.bytesused may have padding bytes for alignment
           Search for EOF to get exact size */
        if (eos_search_size > bytesused)
            eos_search_size = bytesused;
        for (i = 0; i < eos_search_size; i++) {
            p =(uint8_t *)(ctx->g_buff[v4l2_buf.index].start + bytesused);
            if ((*(p-2) == 0xff) && (*(p-1) == 0xd9)) {
                break;
            }
            bytesused--;
        }

        /* Decoding MJPEG frame */
        if (ctx->jpegdec->decodeToFd(fd, ctx->g_buff[v4l2_buf.index].start,
            bytesused, pixfmt, width, height) < 0)
            ERROR_RETURN("Cannot decode MJPEG");

        /* Convert the decoded buffer to YUV420P */
        if (-1 == NvBufferTransform(fd, ctx->render_dmabuf_fd,
                &transParams))
            ERROR_RETURN("Failed to convert the buffer");
    } else {
        if (ctx->capture_dmabuf) {
            /* Cache sync for VIC operation since the data is from CPU */
            NvBufferMemSyncForDevice(ctx->g_buff[v4l2_buf.index].dmabuff_fd, 0,
                    (void**)&ctx->g_buff[v4l2_buf.index].start);
        } else {
            /* Copies raw buffer plane contents to an NvBuffer plane */
            Raw2NvBuffer(ctx->g_buff[v4l2_buf.index].start, 0,
                     ctx->cam_w, ctx->cam_h, ctx->g_buff[v4l2_buf.index].dmabuff_fd);
        }

        /*  Convert the camera buffer from YUV422 to YUV420P */
        if (-1 == NvBufferTransform(ctx->g_buff[v4l2_buf.index].dmabuff_fd, ctx->render_dmabuf_fd,
                    &transParams))
            ERROR_RETURN("Failed to convert the buffer");

        if (ctx->cam_pixfmt == V4L2_PIX_FMT_GREY) {
            if(!nvbuff_do_clearchroma(ctx->render_dmabuf_fd))
                ERROR_RETURN("Failed to clear chroma");
        }
    }
    cuda_postprocess(ctx, ctx->render_dmabuf_fd);

    /* Preview */
    ctx->renderer->render(ctx->render_dmabuf_fd);

    /* Enqueue camera buffer back to driver */
    if (ioctl(ctx->cam_fd, VIDIOC_QBUF, &v4l2_buf))
        ERROR_RETURN("Failed to queue camera buffers: %s (%d)",
                strerror(errno), errno);

}

I’m watching for this URL which looks like the sample code for getting multi plannar from camera.
https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dmabuf.html#example-3-6-queueing-dmabuf-using-multi-plane-api

But my program says

Failed to query buff: Invalid argument (22)

I changed
rb.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
to
rb.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;

at request_camera_buff.

static bool
request_camera_buff(context_t *ctx)
{
    /* Request camera v4l2 buffer */
    struct v4l2_requestbuffers rb;
    memset(&rb, 0, sizeof(rb));
    rb.count = V4L2_BUFFERS_NUM;
    rb.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    rb.memory = V4L2_MEMORY_DMABUF;
    if (ioctl(ctx->cam_fd, VIDIOC_REQBUFS, &rb) < 0)
        ERROR_RETURN("Failed to request v4l2 buffers: %s (%d)",
                strerror(errno), errno);
    if (rb.count != V4L2_BUFFERS_NUM)
        ERROR_RETURN("V4l2 buffer number is not as desired");

    for (unsigned int index = 0; index < V4L2_BUFFERS_NUM; index++)
    {
        struct v4l2_buffer buf;

        /* Query camera v4l2 buf length */
        memset(&buf, 0, sizeof buf);
        buf.index = index;
        buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        buf.memory = V4L2_MEMORY_DMABUF;

        if (ioctl(ctx->cam_fd, VIDIOC_QUERYBUF, &buf) < 0)
            ERROR_RETURN("Failed to query buff: %s (%d)",
                    strerror(errno), errno);

        /* TODO: add support for multi-planer
           Enqueue empty v4l2 buff into camera capture plane */
        buf.m.fd = (unsigned long)ctx->g_buff[index].dmabuff_fd;
        if (buf.length != ctx->g_buff[index].size)
        {
            WARN("Camera v4l2 buf length is not expected");
            ctx->g_buff[index].size = buf.length;
        }

        if (ioctl(ctx->cam_fd, VIDIOC_QBUF, &buf) < 0)
            ERROR_RETURN("Failed to enqueue buffers: %s (%d)",
                    strerror(errno), errno);
    }

    return true;
}

Hi,
So in our 12_camera_v4l2_cuda, the supported formats are all single planar. For multi-planar formats, would need other users to share experience.

Instead of setting V4L2_MEMORY_DMABUF, are you able to capture YUV420 in

v4l2_buf.memory = V4L2_MEMORY_MMAP;

If it woks, you can put into NVvBuffer by calling:

/* Copies raw buffer plane contents to an NvBuffer plane */
Raw2NvBuffer(ctx->g_buff[v4l2_buf.index].start, 0,
         ctx->cam_w, ctx->cam_h, ctx->g_buff[v4l2_buf.index].dmabuff_fd);

Although it takes certain CPU usage.