/* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of NVIDIA CORPORATION nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include "NvEglRenderer.h" #include "NvUtils.h" #include "NvCudaProc.h" #include "nvbuf_utils.h" #include "camera_v4l2_cuda.h" #define OUTPLANE_BUFNUM 6 static bool quit = false; static int outplane_fd[OUTPLANE_BUFNUM]; using namespace std; static void print_usage(void) { printf("\n\tUsage: camera_v4l2_cuda [OPTIONS]\n\n" "\tExample: \n" "\t./camera_v4l2_cuda -d /dev/video0 -s 640x480 -f YUYV -n 30 -c\n\n" "\tSupported options:\n" "\t-d\t\tSet V4l2 video device node\n" "\t-s\t\tSet output resolution of video device\n" "\t-f\t\tSet output pixel format of video device (supports only YUYV/YVYU/UYVY/VYUY)\n" "\t-r\t\tSet renderer frame rate (30 fps by default)\n" "\t-n\t\tSave the n-th frame before VIC processing\n" "\t-c\t\tEnable CUDA aglorithm (draw a black box in the upper left corner)\n" "\t-v\t\tEnable verbose message\n" "\t-h\t\tPrint this usage\n\n" "\tNOTE: It runs infinitely until you terminate it with \n"); } static bool parse_cmdline(context_t * ctx, int argc, char **argv) { int c; if (argc < 2) { print_usage(); exit(EXIT_SUCCESS); } while ((c = getopt(argc, argv, "d:s:f:r:n:cvh")) != -1) { switch (c) { case 'd': ctx->cam_devname = optarg; break; case 's': if (sscanf(optarg, "%dx%d", &ctx->cam_w, &ctx->cam_h) != 2) { print_usage(); return false; } break; case 'f': if (strcmp(optarg, "YUYV") == 0) ctx->cam_pixfmt = V4L2_PIX_FMT_YUYV; else if (strcmp(optarg, "YVYU") == 0) ctx->cam_pixfmt = V4L2_PIX_FMT_YVYU; else if (strcmp(optarg, "VYUY") == 0) ctx->cam_pixfmt = V4L2_PIX_FMT_VYUY; else if (strcmp(optarg, "UYVY") == 0) ctx->cam_pixfmt = V4L2_PIX_FMT_UYVY; else { print_usage(); return false; } sprintf(ctx->cam_file, "camera.%s", optarg); break; case 'r': ctx->fps = strtol(optarg, NULL, 10); break; case 'n': ctx->save_n_frame = strtol(optarg, NULL, 10); break; case 'c': ctx->enable_cuda = true; break; case 'v': ctx->enable_verbose = true; break; case 'h': print_usage(); exit(EXIT_SUCCESS); break; default: print_usage(); return false; } } return true; } static void set_defaults(context_t * ctx) { memset(ctx, 0, sizeof(context_t)); ctx->cam_devname = "/dev/video0"; ctx->cam_fd = -1; ctx->cam_pixfmt = V4L2_PIX_FMT_YUYV; ctx->cam_w = 640; ctx->cam_h = 480; ctx->frame = 0; ctx->save_n_frame = 0; ctx->g_buff = NULL; ctx->renderer = NULL; ctx->fps = 30; ctx->enable_cuda = false; ctx->egl_image = NULL; ctx->egl_display = EGL_NO_DISPLAY; ctx->enable_verbose = false; } static nv_color_fmt nvcolor_fmt[] = { // TODO add more pixel format mapping {V4L2_PIX_FMT_UYVY, NvBufferColorFormat_UYVY}, {V4L2_PIX_FMT_VYUY, NvBufferColorFormat_VYUY}, {V4L2_PIX_FMT_YUYV, NvBufferColorFormat_YUYV}, {V4L2_PIX_FMT_YVYU, NvBufferColorFormat_YVYU}, {V4L2_PIX_FMT_YUV420M, NvBufferColorFormat_YUV420}, }; static NvBufferColorFormat get_nvbuff_color_fmt(unsigned int v4l2_pixfmt) { unsigned i; for (i = 0; i < sizeof(nvcolor_fmt) / sizeof(nvcolor_fmt[0]); i++) { if (v4l2_pixfmt == nvcolor_fmt[i].v4l2_pixfmt) return nvcolor_fmt[i].nvbuff_color; } return NvBufferColorFormat_Invalid; } static bool save_frame_to_file(context_t * ctx, struct v4l2_buffer * buf) { int file; file = open(ctx->cam_file, O_CREAT | O_WRONLY | O_APPEND | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); if (-1 == file) ERROR_RETURN("Failed to open file for frame saving"); if (-1 == write(file, ctx->g_buff[buf->index].start, ctx->g_buff[buf->index].size)) { close(file); ERROR_RETURN("Failed to write frame into file"); } close(file); return true; } static bool camera_initialize(context_t * ctx) { struct v4l2_format fmt; // Open camera device ctx->cam_fd = open(ctx->cam_devname, O_RDWR); if (ctx->cam_fd == -1) ERROR_RETURN("Failed to open camera device %s: %s (%d)", ctx->cam_devname, strerror(errno), errno); // Set camera output format memset(&fmt, 0, sizeof(fmt)); fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; fmt.fmt.pix.width = ctx->cam_w; fmt.fmt.pix.height = ctx->cam_h; fmt.fmt.pix.pixelformat = ctx->cam_pixfmt; fmt.fmt.pix.field = V4L2_FIELD_INTERLACED; if (ioctl(ctx->cam_fd, VIDIOC_S_FMT, &fmt) < 0) ERROR_RETURN("Failed to set camera output format: %s (%d)", strerror(errno), errno); // Get the real format in case the desired is not supported memset(&fmt, 0, sizeof fmt); fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; if (ioctl(ctx->cam_fd, VIDIOC_G_FMT, &fmt) < 0) ERROR_RETURN("Failed to get camera output format: %s (%d)", strerror(errno), errno); if (fmt.fmt.pix.width != ctx->cam_w || fmt.fmt.pix.height != ctx->cam_h || fmt.fmt.pix.pixelformat != ctx->cam_pixfmt) { WARN("The desired format is not supported"); ctx->cam_w = fmt.fmt.pix.width; ctx->cam_h = fmt.fmt.pix.height; ctx->cam_pixfmt =fmt.fmt.pix.pixelformat; } INFO("Camera ouput format: (%d x %d) stride: %d, imagesize: %d", fmt.fmt.pix.width, fmt.fmt.pix.height, fmt.fmt.pix.bytesperline, fmt.fmt.pix.sizeimage); return true; } static bool display_initialize(context_t * ctx) { // Create EGL renderer ctx->renderer = NvEglRenderer::createEglRenderer("renderer0", ctx->cam_w, ctx->cam_h, 0, 0); if (!ctx->renderer) ERROR_RETURN("Failed to create EGL renderer"); ctx->renderer->setFPS(ctx->fps); if (ctx->enable_cuda) { // Get defalut EGL display ctx->egl_display = eglGetDisplay(EGL_DEFAULT_DISPLAY); if (ctx->egl_display == EGL_NO_DISPLAY) ERROR_RETURN("Failed to get EGL display connection"); // Init EGL display connection if (!eglInitialize(ctx->egl_display, NULL, NULL)) ERROR_RETURN("Failed to initialize EGL display connection"); } return true; } static bool encoder_initialize(context_t * ctx) { ctx->enc = NvVideoEncoder::createVideoEncoder("enc0"); if (ctx->enc == NULL) ERROR_RETURN("Failed to create video encoder"); if (ctx->enc->setCapturePlaneFormat(V4L2_PIX_FMT_H264, ctx->cam_w, ctx->cam_h, 2 * 1024 * 1024) < 0) ERROR_RETURN("Failed to set up ENC capture plane format"); if (ctx->enc->setOutputPlaneFormat(V4L2_PIX_FMT_YUV420M, ctx->cam_w, ctx->cam_h) < 0) ERROR_RETURN("Failed to set up ENC output plane format"); if (ctx->enc->setBitrate(4<<20) < 0) ERROR_RETURN("Failed to set up ENC bitrate"); if (ctx->enc->setProfile(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH) < 0) ERROR_RETURN("Failed to set up ENC profile"); if (ctx->enc->setLevel(V4L2_MPEG_VIDEO_H264_LEVEL_5_0) < 0) ERROR_RETURN("Failed to set up ENC level"); if (ctx->enc->setRateControlMode(V4L2_MPEG_VIDEO_BITRATE_MODE_VBR) < 0) ERROR_RETURN("Failed to set up ENC rate control mode"); if (ctx->enc->output_plane.setupPlane(V4L2_MEMORY_DMABUF, OUTPLANE_BUFNUM, false, false) < 0) ERROR_RETURN("Failed to set up ENC output plane"); if (ctx->enc->capture_plane.setupPlane(V4L2_MEMORY_MMAP, 10, true, false) < 0) ERROR_RETURN("Failed to set up ENC capture plane"); ctx->enc->subscribeEvent(V4L2_EVENT_EOS,0,0); ctx->enc->setEncoderCommand(V4L2_ENC_CMD_START, 0); return true; } static bool init_components(context_t * ctx) { if (!camera_initialize(ctx)) ERROR_RETURN("Failed to initialize camera device"); if (!display_initialize(ctx)) ERROR_RETURN("Failed to initialize display"); if (!encoder_initialize(ctx)) ERROR_RETURN("Failed to initialize encoder"); INFO("Initialize v4l2 components successfully"); return true; } static bool request_camera_buff(context_t *ctx) { // Request camera v4l2 buffer struct v4l2_requestbuffers rb; memset(&rb, 0, sizeof(rb)); rb.count = V4L2_BUFFERS_NUM; rb.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; rb.memory = V4L2_MEMORY_DMABUF; if (ioctl(ctx->cam_fd, VIDIOC_REQBUFS, &rb) < 0) ERROR_RETURN("Failed to request v4l2 buffers: %s (%d)", strerror(errno), errno); if (rb.count != V4L2_BUFFERS_NUM) ERROR_RETURN("V4l2 buffer number is not as desired"); for (unsigned int index = 0; index < V4L2_BUFFERS_NUM; index++) { struct v4l2_buffer buf; // Query camera v4l2 buf length memset(&buf, 0, sizeof buf); buf.index = index; buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; buf.memory = V4L2_MEMORY_DMABUF; if (ioctl(ctx->cam_fd, VIDIOC_QUERYBUF, &buf) < 0) ERROR_RETURN("Failed to query buff: %s (%d)", strerror(errno), errno); // TODO add support for multi-planer // Enqueue empty v4l2 buff into camera capture plane buf.m.fd = (unsigned long)ctx->g_buff[index].dmabuff_fd; if (buf.length != ctx->g_buff[index].size) { WARN("Camera v4l2 buf length is not expected"); ctx->g_buff[index].size = buf.length; } if (ioctl(ctx->cam_fd, VIDIOC_QBUF, &buf) < 0) ERROR_RETURN("Failed to enqueue buffers: %s (%d)", strerror(errno), errno); } return true; } static bool prepare_buffers(context_t * ctx) { NvBufferCreateParams input_params = {0}; // Allocate global buffer context ctx->g_buff = (nv_buffer *)malloc(V4L2_BUFFERS_NUM * sizeof(nv_buffer)); if (ctx->g_buff == NULL) ERROR_RETURN("Failed to allocate global buffer context"); input_params.payloadType = NvBufferPayload_SurfArray; input_params.width = ctx->cam_w; input_params.height = ctx->cam_h; input_params.layout = NvBufferLayout_Pitch; // Create buffer and provide it with camera for (unsigned int index = 0; index < V4L2_BUFFERS_NUM; index++) { int fd; NvBufferParams params = {0}; input_params.colorFormat = get_nvbuff_color_fmt(ctx->cam_pixfmt); input_params.nvbuf_tag = NvBufferTag_CAMERA; if (-1 == NvBufferCreateEx(&fd, &input_params)) ERROR_RETURN("Failed to create NvBuffer"); ctx->g_buff[index].dmabuff_fd = fd; if (-1 == NvBufferGetParams(fd, ¶ms)) ERROR_RETURN("Failed to get NvBuffer parameters"); // TODO add multi-planar support // Currently it supports only YUV422 interlaced single-planar if (-1 == NvBufferMemMap(ctx->g_buff[index].dmabuff_fd, 0, NvBufferMem_Read_Write, (void**)&ctx->g_buff[index].start)) ERROR_RETURN("Failed to map buffer"); } input_params.colorFormat = get_nvbuff_color_fmt(V4L2_PIX_FMT_YUV420M); input_params.nvbuf_tag = NvBufferTag_NONE; // Create Render buffer if (-1 == NvBufferCreateEx(&ctx->render_dmabuf_fd, &input_params)) ERROR_RETURN("Failed to create NvBuffer"); if (!request_camera_buff(ctx)) ERROR_RETURN("Failed to set up camera buff"); INFO("Succeed in preparing stream buffers"); return true; } static bool start_stream(context_t * ctx) { enum v4l2_buf_type type; // Start v4l2 streaming type = V4L2_BUF_TYPE_VIDEO_CAPTURE; if (ioctl(ctx->cam_fd, VIDIOC_STREAMON, &type) < 0) ERROR_RETURN("Failed to start streaming: %s (%d)", strerror(errno), errno); // Start ENC output plane if (ctx->enc->output_plane.setStreamStatus(true) < 0) ERROR_RETURN("Failed to start ENC output plane streaming"); // Start ENC capture plane if (ctx->enc->capture_plane.setStreamStatus(true) < 0) ERROR_RETURN("Failed to start ENC capture plane streaming"); usleep(200); INFO("Camera video streaming on ..."); return true; } static bool enc_capture_dqbuf_thread_callback(struct v4l2_buffer *v4l2_buf, NvBuffer * buffer, NvBuffer * shared_buffer, void *arg) { context_t *ctx = (context_t *) arg; if (v4l2_buf == NULL) { printf("Got nullptr \n"); return false; } if(v4l2_buf->flags & V4L2_BUF_FLAG_LAST) { struct v4l2_event ev; int ret = 0; memset(&ev,0,sizeof(struct v4l2_event)); ret = ctx->enc->dqEvent(ev,1000); if (ret < 0) printf("Error in dqEvent \n");; if(ev.type == V4L2_EVENT_EOS) { printf("Got EOS, exiting...\n"); return false; } } printf("encoded frame size %d \n", buffer->planes[0].bytesused); if (ctx->enc->capture_plane.qBuffer(*v4l2_buf, NULL) < 0) { ctx->enc->abort(); ERROR_RETURN("Failed to queue buffer on ENC capture plane"); } return true; } static void signal_handle(int signum) { printf("Quit due to exit command from user!\n"); quit = true; } static bool cuda_postprocess(context_t *ctx, int fd) { if (ctx->enable_cuda) { // Create EGLImage from dmabuf fd ctx->egl_image = NvEGLImageFromFd(ctx->egl_display, fd); if (ctx->egl_image == NULL) ERROR_RETURN("Failed to map dmabuf fd (0x%X) to EGLImage", ctx->render_dmabuf_fd); // Running algo process with EGLImage via GPU multi cores HandleEGLImage(&ctx->egl_image); // Destroy EGLImage NvDestroyEGLImage(ctx->egl_display, ctx->egl_image); ctx->egl_image = NULL; } return true; } static bool start_capture(context_t * ctx) { struct sigaction sig_action; struct pollfd fds[1]; NvBufferTransformParams transParams; // Ensure a clean shutdown if user types sig_action.sa_handler = signal_handle; sigemptyset(&sig_action.sa_mask); sig_action.sa_flags = 0; sigaction(SIGINT, &sig_action, NULL); ctx->enc->capture_plane.setDQThreadCallback(enc_capture_dqbuf_thread_callback); ctx->enc->capture_plane.startDQThread(ctx); // Enqueue all the empty capture plane buffers for (uint32_t i = 0; i < ctx->enc->capture_plane.getNumBuffers(); i++) { struct v4l2_buffer v4l2_buf; struct v4l2_plane planes[MAX_PLANES]; memset(&v4l2_buf, 0, sizeof(v4l2_buf)); memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); v4l2_buf.index = i; v4l2_buf.m.planes = planes; if (ctx->enc->capture_plane.qBuffer(v4l2_buf, NULL) < 0) ERROR_RETURN("Failed to queue buffer on ENC capture plane"); } // Init the NvBufferTransformParams memset(&transParams, 0, sizeof(transParams)); transParams.transform_flag = NVBUFFER_TRANSFORM_FILTER; transParams.transform_filter = NvBufferTransform_Filter_Smart; // Enable render profiling information ctx->renderer->enableProfiling(); int bufferIndex = 0; NvBufferCreateParams input_params = {0}; input_params.payloadType = NvBufferPayload_SurfArray; input_params.width = ctx->cam_w; input_params.height = ctx->cam_h; input_params.layout = NvBufferLayout_Pitch; input_params.colorFormat = get_nvbuff_color_fmt(V4L2_PIX_FMT_YUV420M); input_params.nvbuf_tag = NvBufferTag_VIDEO_ENC; fds[0].fd = ctx->cam_fd; fds[0].events = POLLIN; while (poll(fds, 1, 5000) > 0 && !quit) { if (fds[0].revents & POLLIN) { struct v4l2_buffer v4l2_buf; // Dequeue camera buff memset(&v4l2_buf, 0, sizeof(v4l2_buf)); v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; v4l2_buf.memory = V4L2_MEMORY_DMABUF; if (ioctl(ctx->cam_fd, VIDIOC_DQBUF, &v4l2_buf) < 0) ERROR_RETURN("Failed to dequeue camera buff: %s (%d)", strerror(errno), errno); ctx->frame++; if (ctx->frame == ctx->save_n_frame) save_frame_to_file(ctx, &v4l2_buf); // Cache sync for VIC operation NvBufferMemSyncForDevice(ctx->g_buff[v4l2_buf.index].dmabuff_fd, 0, (void**)&ctx->g_buff[v4l2_buf.index].start); #if 1 NvBuffer *buffer; int fd = -1; struct v4l2_buffer enc_buf; struct v4l2_plane planes[MAX_PLANES]; memset(&enc_buf, 0, sizeof(enc_buf)); memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); enc_buf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT; enc_buf.memory = V4L2_MEMORY_DMABUF; enc_buf.m.planes = planes; // Check if we need dqBuffer first if (bufferIndex < OUTPLANE_BUFNUM && ctx->enc->output_plane.getNumQueuedBuffers() < ctx->enc->output_plane.getNumBuffers()) { // The queue is not full, no need to dqBuffer // Prepare buffer index for the following qBuffer enc_buf.index = bufferIndex; // Create Render buffer if (-1 == NvBufferCreateEx(&fd, &input_params)) ERROR_RETURN("Failed to create NvBuffer"); outplane_fd[bufferIndex] = fd; bufferIndex++;; } else { // Output plane full or max outstanding number reached ctx->enc->output_plane.dqBuffer(enc_buf, &buffer, NULL, 10); fd = enc_buf.m.planes[0].m.fd; } if (-1 == NvBufferTransform(ctx->g_buff[v4l2_buf.index].dmabuff_fd, fd, &transParams)) ERROR_RETURN("Failed to convert the buffer"); // Push the frame into V4L2. enc_buf.m.planes[0].m.fd = fd; enc_buf.m.planes[0].bytesused = 1; // byteused must be non-zero ctx->enc->output_plane.qBuffer(enc_buf, NULL); ctx->renderer->render(fd); #else // Convert the camera buffer from YUV422 to YUV420P if (-1 == NvBufferTransform(ctx->g_buff[v4l2_buf.index].dmabuff_fd, ctx->render_dmabuf_fd, &transParams)) ERROR_RETURN("Failed to convert the buffer"); cuda_postprocess(ctx, ctx->render_dmabuf_fd); ctx->renderer->render(ctx->render_dmabuf_fd); #endif // Enqueue camera buff if (ioctl(ctx->cam_fd, VIDIOC_QBUF, &v4l2_buf)) ERROR_RETURN("Failed to queue camera buffers: %s (%d)", strerror(errno), errno); } } // Print profiling information when streaming stops. ctx->renderer->printProfilingStats(); return true; } static bool stop_stream(context_t * ctx) { enum v4l2_buf_type type; // Stop v4l2 streaming type = V4L2_BUF_TYPE_VIDEO_CAPTURE; if (ioctl(ctx->cam_fd, VIDIOC_STREAMOFF, &type)) ERROR_RETURN("Failed to stop streaming: %s (%d)", strerror(errno), errno); ctx->enc->setEncoderCommand(V4L2_ENC_CMD_STOP, 1); ctx->enc->capture_plane.waitForDQThread(2000); INFO("Camera video streaming off ..."); return true; } int main(int argc, char *argv[]) { context_t ctx; int error = 0, i; set_defaults(&ctx); CHECK_ERROR(parse_cmdline(&ctx, argc, argv), cleanup, "Invalid options specified"); CHECK_ERROR(init_components(&ctx), cleanup, "Failed to initialize v4l2 components"); CHECK_ERROR(prepare_buffers(&ctx), cleanup, "Failed to prepare v4l2 buffs"); CHECK_ERROR(start_stream(&ctx), cleanup, "Failed to start streaming"); CHECK_ERROR(start_capture(&ctx), cleanup, "Failed to start capturing") CHECK_ERROR(stop_stream(&ctx), cleanup, "Failed to stop streaming"); cleanup: if (ctx.cam_fd > 0) close(ctx.cam_fd); if (ctx.renderer != NULL) delete ctx.renderer; if (ctx.egl_display && !eglTerminate(ctx.egl_display)) printf("Failed to terminate EGL display connection\n"); if (ctx.g_buff != NULL) { for (unsigned i = 0; i < V4L2_BUFFERS_NUM; i++) if (ctx.g_buff[i].dmabuff_fd) NvBufferDestroy(ctx.g_buff[i].dmabuff_fd); free(ctx.g_buff); } NvBufferDestroy(ctx.render_dmabuf_fd); for (i = 0; i < OUTPLANE_BUFNUM ; i++) NvBufferDestroy(outplane_fd[i]); if (ctx.enc != NULL) delete ctx.enc; if (error) printf("App run failed\n"); else printf("App run was successful\n"); return -error; }