capture + encode on TX2

HI nividia,

I want to capture and encode , i tired to combine 12th (capturing) and 1st(encoding) example but getting crash or hang problem .

Problem is

  1. how to fill the capture data into encoder buffer.
  2. Need to replace read_video_frame.
  3. In read_video_frame function , Nv_Buffer is getting used but capture code having data in v4l2_buf buffer.

Do you have any example which captures and encodes for TX2 Jetpack 3.2

below is the core code:

ctx->conv->capture_plane.setDQThreadCallback(conv_capture_dqbuf_thread_callback);
ctx->conv->output_plane.setDQThreadCallback(conv_output_dqbuf_thread_callback);


//Encoder//
ret = pVideoAudioEncode->VCE_InitEncConfig(pstEncCtx);
if(ret == FAILURE)
{
	goto cleanup;
}


pstEncCtx->enc->capture_plane.setDQThreadCallback(encoder_capture_plane_dq_callback);


//Capture//
// Start VIC processing thread
ctx->conv->capture_plane.startDQThread(ctx);
ctx->conv->output_plane.startDQThread(ctx);


// startDQThread starts a thread internally which calls the
// encoder_capture_plane_dq_callback whenever a buffer is dequeued
// on the plane
pstEncCtx->enc->capture_plane.startDQThread(pstEncCtx);



ret = pVideoAudioEncode->VCE_EnqueueEmptyCapturePlane(pstEncCtx);
if(ret == FAILURE)
{
	goto cleanup;
}


// Enable render profiling information
//ctx->renderer->enableProfiling();

fds[0].fd = ctx->cam_fd;
fds[0].events = POLLIN;
while (poll(fds, 1, 5000) > 0 && !ctx->got_error &&
		!ctx->conv->isInError() && !quit)
{
	if (fds[0].revents & POLLIN)
	{
		struct v4l2_buffer v4l2_buf;

		// Dequeue camera buff
		memset(&v4l2_buf, 0, sizeof(v4l2_buf));
		v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
		v4l2_buf.memory = V4L2_MEMORY_DMABUF;
		if (ioctl(ctx->cam_fd, VIDIOC_DQBUF, &v4l2_buf) < 0)
			ERROR_RETURN("Failed to dequeue camera buff: %s (%d)",
					strerror(errno), errno);

		ctx->frame++;
		printf("\n frame No = %d \n",ctx->frame);

		if (ctx->frame == ctx->save_n_frame)
			save_frame_to_file(ctx, &v4l2_buf);

		// Push nv_buffer into conv output queue for conversion
		pthread_mutex_lock(&ctx->queue_lock);
		ctx->conv_output_plane_buf_queue->push(&ctx->g_buff[v4l2_buf.index]);
		pthread_cond_broadcast(&ctx->queue_cond);
		pthread_mutex_unlock(&ctx->queue_lock);

		//Encoder//
		pVideoAudioEncode->VCE_ReadAndQueueOuputPlanesBuff(&eos, &input_frames_queued_count,&v4l2_buf);

		// Enqueue camera buff
		// It might be more reasonable to wait for the completion of
		// VIC processing before enqueue current buff. But VIC processing
		// time is far less than camera frame interval, so we probably
		// don't need such synchonization.
		if (ioctl(ctx->cam_fd, VIDIOC_QBUF, &v4l2_buf))
			ERROR_RETURN("Failed to queue camera buffers: %s (%d)",
					strerror(errno), errno);
	}
}

if (quit && !ctx->conv->isInError())
{
	// Signal EOS to the dq thread of VIC output plane
	ctx->g_buff[0].dmabuff_fd = 0;

	pthread_mutex_lock(&ctx->queue_lock);
	ctx->conv_output_plane_buf_queue->push(&ctx->g_buff[0]);
	pthread_cond_broadcast(&ctx->queue_cond);
	pthread_mutex_unlock(&ctx->queue_lock);
}

// Stop VIC dq thread
if (!ctx->got_error)
{
	ctx->conv->output_plane.deinitPlane();
	ctx->conv->capture_plane.deinitPlane();
}

VCE_ReadAndQueueOuputPlanesBuff(bool *bEos, unsigned int *pnInputFramesQueuedCount,struct v4l2_buffer *pv4l2_buffer)
{
int ret = SUCCESS;
Enccontext_t *pstEncCtx = m_pstEncCtx;
int i;

// Read video frame and queue all the output plane buffers +
// Keep reading input till EOS is reached
while (!pstEncCtx->got_error && !pstEncCtx->enc->isInError() && !*bEos)
{
	struct v4l2_buffer v4l2_buf;
	struct v4l2_plane planes[MAX_PLANES];
	NvBuffer *buffer ;

	memset(&v4l2_buf, 0, sizeof(v4l2_buf));
	memset(planes, 0, sizeof(planes));

	v4l2_buf.m.planes = planes;

	if(*pnInputFramesQueuedCount < pstEncCtx->enc->output_plane.getNumBuffers())
	{
		v4l2_buf.index = *pnInputFramesQueuedCount;
		buffer = pstEncCtx->enc->output_plane.getNthBuffer(*pnInputFramesQueuedCount);
	}
	else
	{
		// need to fill data
		if (pstEncCtx->enc->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0)
		{
			cerr << "ERROR while DQing buffer at output plane" << endl;
			VCE_Abort(pstEncCtx);
			goto cleanup;
		}
	}


	if (pstEncCtx->runtime_params_str &&
			(pstEncCtx->enc->output_plane.getTotalQueuedBuffers() ==
					pstEncCtx->next_param_change_frame))
	{
		VCE_SetRuntimeParams(pstEncCtx);
		if (pstEncCtx->runtime_params_str)
			VCE_GetNextRuntimeParamChangeFrame(pstEncCtx);
	}

	for(i = 0; i < buffer->n_planes ; i++)
	{
		buffer->planes[i].data = (unsigned char *)pv4l2_buffer->m.planes[i].m.userptr;
		//buffer->planes[i].bytesused = pv4l2_buffer->m.planes[i].bytesused;
		//buffer->planes[i].fd =  pv4l2_buffer->m.planes[i].m.fd;
	}

	printf("\n fill the buffer \n");
			//if (read_video_frame(pstEncCtx->in_file, *buffer) < 0)
			//{
			//	cerr << "Could not read complete frame from input file" << endl;
			//	v4l2_buf.m.planes[0].bytesused = 0;
			//}


	ret = pstEncCtx->enc->output_plane.qBuffer(v4l2_buf, NULL);
	if (ret < 0)
	{
		cerr << "Error while queueing buffer at output plane" << endl;
		VCE_Abort(pstEncCtx);
		goto cleanup;
	}

	if (v4l2_buf.m.planes[0].bytesused == 0)
	{
		cerr << "File read complete." << endl;
		*bEos = true;
		break;
	}
	*pnInputFramesQueuedCount++;
}

ret = SUCCESS;
cleanup:
if(ret != SUCCESS)
{
	ret = FAILURE;
}

return ret;

}

Hi meRaza,

Any reason for not using Gstreamer? A quick way to test capture/encode is using pipelines, you can find some pipelines here:
http://developer.ridgerun.com/wiki/index.php?title=Gstreamer_pipelines_for_Tegra_X2

Hi josejich,

actually , we decided to go with Tegra multimedia APIs approach .

I found code which captures and encodes for TX1 but it gives error on TX2.

./standalone -d /dev/video0 -s 1920x1080 -f UYVY
Failed to query video capabilities: Inappropriate ioctl for device
libv4l2_nvvidconv (0):(792) (INFO) : Allocating (4) OUTPUT PLANE BUFFERS Layout=0
libv4l2_nvvidconv (0):(808) (INFO) : Allocating (4) CAPTURE PLANE BUFFERS Layout=0
Failed to query video capabilities: Inappropriate ioctl for device
NvMMLiteOpen : Block : BlockType = 4
===== MSENC =====
NvMMLiteBlockCreate : Block : BlockType = 4
875967048
842091865
===== MSENC blits (mode: 1) into tiled surfaces =====
[ERROR] (…/classes/NvV4l2ElementPlane.cpp:253) Output Plane:Error while Qing buffer: Device or resource busy
ERROR: conv_capture_dqbuf_thread_callback(): (line:638) Failed to queue buffer on ENC output plane

code
https://devtalk.nvidia.com/cmd/default/download-comment-attachment/72132/

discussing thread
https://devtalk.nvidia.com/default/topic/999493/jetson-tx1/nvidia-multimedia-apis-with-uyvy-sensor/1

can anyone help in solving this ? thanks in advance

Hi meRaza,
The sample is for r24.2.1. You may refer to the code diff and adapt to 12_camera_v4l2_cuda of r28.2 DP

HI Danel,

I tried to check the difference but cant find the similarities b/n jetpack 3.0 r24.2.1 and jetpack 3.2 r28.2.
Kindly can you help in adapting the 12_camera_v4l2_cuda to r28.2 .

Regards,
Aasim

Hi Danel,

I went through your previous post for capture + jpeg encoding
{ Refer to 09_camera_jpeg_capture and pass (Fd) to jpeg encoder. The flow is V4L2 camera -> NvBuffer(fd) -> jpeg encoder }

previous post

https://devtalk.nvidia.com/default/topic/984850/jetson-tx1/how-to-convert-yuv-to-jpg-using-jpeg-encoder-hardware-/post/5048479/#5048479

As per your current suggestion i compared the code with latest 12_camera_v4l2_cuda of r28.2 DP.
The main difference i found in conv_capture_dqbuf_thread_callback() function , where this data flow happens { V4L2 camera -> NvBuffer(fd) -> encoder} enc->output_plane.qBuffer

1) I am facing problem in transferring data from capture to encoder and getting this error.

[ERROR] (../classes/NvV4l2ElementPlane.cpp:253) <enc0> Output Plane:Error while Qing buffer: Device or resource busy
ERROR: conv_capture_dqbuf_thread_callback(): (line:649) Failed to queue buffer on ENC output plane

code of conv_capture_dqbuf_thread_callback() function

static bool
conv_capture_dqbuf_thread_callback(struct v4l2_buffer *v4l2_buf,
                                   NvBuffer * buffer, NvBuffer * shared_buffer,
                                   void *arg)
{
    context_t *ctx = (context_t *) arg;

    if (ctx->enable_cuda)
    {
        // Create EGLImage from dmabuf fd
        ctx->egl_image = NvEGLImageFromFd(ctx->egl_display, buffer->planes[0].fd);
        if (ctx->egl_image == NULL)
            ERROR_RETURN("Failed to map dmabuf fd (0x%X) to EGLImage",
                    buffer->planes[0].fd);

        // Running algo process with EGLImage via GPU multi cores
        HandleEGLImage(&ctx->egl_image);

        // Destroy EGLImage
        NvDestroyEGLImage(ctx->egl_display, ctx->egl_image);
        ctx->egl_image = NULL;
    }

    struct v4l2_buffer enc_buf;
    struct v4l2_plane planes[MAX_PLANES];

    memset(&enc_buf, 0, sizeof(v4l2_buf));
    memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane));

    enc_buf.m.planes = planes;
    if (buffer->planes[0].bytesused)
    {
        enc_buf.m.planes[0].m.fd = buffer->planes[0].fd;
        enc_buf.m.planes[0].bytesused = 1;

        if (ctx->enc->output_plane.qBuffer(enc_buf, buffer) < 0)
        {
            abort(ctx);
            ERROR_RETURN("Failed to queue buffer on ENC output plane");
        }
    }

// Render the frame into display
    if (v4l2_buf->m.planes[0].bytesused && ctx->renderer)
       ctx->renderer->render(buffer->planes[0].fd);

    if (ctx->conv->capture_plane.qBuffer(*v4l2_buf, buffer) < 0)
    {
        abort(ctx);
        ERROR_RETURN("Failed to queue buffer on VIC capture plane");
    }

    return true;
}

2) can you help me in achieving this data flow right { V4L2 camera -> NvBuffer(fd) -> encoder}

This is the code ?/ or any other way? to transfer the data to encoder output plane

struct v4l2_buffer enc_buf;
    struct v4l2_plane planes[MAX_PLANES];

    memset(&enc_buf, 0, sizeof(v4l2_buf));
    memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane));

    enc_buf.m.planes = planes;
    if (buffer->planes[0].bytesused)
    {
        enc_buf.m.planes[0].m.fd = buffer->planes[0].fd;
        enc_buf.m.planes[0].bytesused = 1;

        if (ctx->enc->output_plane.qBuffer(enc_buf, buffer) < 0)
        {
            abort(ctx);
            ERROR_RETURN("Failed to queue buffer on ENC output plane");
        }
    }

Hi meRaza,

You can refer my thread with source code inside
https://devtalk.nvidia.com/default/topic/1025242/jetson-tx2/tegra-multimedia-apis-could-not-get-raw-image-via-v4l2-as-well/1
I capture frames from camera, put them to video converter and finally to encoder.

Hi forever3000,

Thanks dude for post , i missed to incrementing the buffer index and limit it 6.

The problem is solved …

one doubt i have , how you have handled the audio part ?

Hi meRaza,

I didn’t try with audio. You might need support from NVIDIA.

sure forever3000, i will create a new thread for audio question … thanks for your grate help it saved my much time.