Hi,
I have a camera that outputs UYVY format at 1920x1020 resolution, and a camera driver that pulls frames from the camera at 10 FPS. I am running this on my AGX Orin, and I would like to use the Jetson Multimedia API to leverage hardware acceleration for transforming the images from UYVY to YUV420 and subsequently encode the YUV420 images into JPEG. I am aware of a similar issue here; however, the API has changed quite a bit since this post.
The issue I am currently facing is that after transforming from UYVY to YUV420 and encoding into JPEG, the JPEG images look like this (see attachment below). It looks like the JPEG image is roughly correct, but properties of the pixels are messed up.
I have attached a condensed snippet of my camera driver. Note that I am using the Linux V4L2 library to talk to my camera. Also, I have mainly referenced the 05_jpeg_encode sample when writing my code. I have also verified that the camera preview when running 12_camera_v4l2_cuda works as expected for my camera, as I am able to see clear images from my camera with ./v4l2_camera_cude -d /dev/video0 -f UYVY.
// Adapted from read_dmabuf() in NvUtils.h to use std::vector<std::byte> instead of std::ifstream*
int read_dmabuf_byte_stream(int dmabuf_fd, unsigned int plane, const std::vector<std::byte>& buffer) {
if (dmabuf_fd <= 0 || buffer.empty()) {
return -1;
}
int ret = -1;
NvBufSurface *nvbuf_surf = nullptr;
ret = NvBufSurfaceFromFd(dmabuf_fd, (void**)(&nvbuf_surf));
if (ret != 0) {
return -1;
}
NvBufSurfaceMap(nvbuf_surf, 0, plane, NVBUF_MAP_READ_WRITE);
NvBufSurfaceSyncForCpu(nvbuf_surf, 0, plane);
for (uint i = 0; i < nvbuf_surf->surfaceList->planeParams.height[plane]; ++i) {
memcpy((char*)nvbuf_surf->surfaceList->mappedAddr.addr[plane] + i * nvbuf_surf->surfaceList->planeParams.pitch[plane],
buffer.data() + i * nvbuf_surf->surfaceList->planeParams.width[plane] * nvbuf_surf->surfaceList->planeParams.bytesPerPix[plane],
nvbuf_surf->surfaceList->planeParams.width[plane] * nvbuf_surf->surfaceList->planeParams.bytesPerPix[plane]);
}
NvBufSurfaceSyncForDevice(nvbuf_surf, 0, plane);
ret = NvBufSurfaceUnMap(nvbuf_surf, 0, plane);
if (ret < 0) {
printf("Error while Unmapping buffer\n");
return ret;
}
return 0;
}
absl::Status run_once() {
const int width = 1920;
const int height = 1080;
// Set V4L2 device params
V4L2DeviceParameters params(
"/dev/video0",
V4L2_PIX_FMT_UYVY,
width,
height,
10
);
// Create V4L2 Mmap device
std::unique_ptr<V4l2MmapDevice> camera_ = std::make_unique<V4l2MmapDevice>(params, V4L2_BUF_TYPE_VIDEO_CAPTURE);
// Create CPU buffer for UYVY image
auto buffer_size = camera_->getBufferSize();
std::vector<std::byte> buffer;
buffer.resize(buffer_size);
// Create CPU buffer for JPEG image
unsigned long out_buf_size = height * width * 3 / 2;
std::vector<unsigned char> out_buf(out_buf_size);
unsigned char * out_buf_ptr = &out_buf[0];
// Receive UYVY image from camera
timeval image_ts;
int read_size = camera_->read(reinterpret_cast<char*>(buffer.data()), buffer_size, image_ts);
if (read_size == -1) {
return absl::InternalError("Failed to read image");
} else if (read_size != buffer_size) {
return absl::InternalError("Image incomplete: Read " + std::to_string(read_size) + "but expected " + std::to_string(buffer_size));
}
int ret = 0;
int iterator_num = 1;
int src_dma_fd = -1;
int dst_dma_fd = -1;
// Create Nv encoder
auto jpegenc = NvJPEGEncoder::createJPEGEncoder("jpenenc");
// Set Nv buffer params for UYVY image
NvBufSurf::NvCommonAllocateParams params;
params.memType = NVBUF_MEM_SURFACE_ARRAY;
params.width = width;
params.height = height;
// For when buffer is read or written by CPU
params.layout = NVBUF_LAYOUT_PITCH;
params.colorFormat = NVBUF_COLOR_FORMAT_UYVY;
params.memtag = NvBufSurfaceTag_VIDEO_CONVERT;
// Allocate Nv buffer for UYVY image
ret = NvBufSurf::NvAllocate(¶ms, 1, &src_dma_fd);
// Fill Nv buffer for UYVY image with UYVY image
if (read_dmabuf_byte_stream(src_dma_fd, 0, buffer) != 0) {
std::cerr << "read_dmabuf_byte_stream failed" << std::endl;
return absl::InternalError("read_dmabuf_byte_stream failed");
}
// Set Nv buffer params for YUV420 image
params.memType = NVBUF_MEM_SURFACE_ARRAY;
params.width = width;
params.height = height;
// For when buffer is used by hardware acceleration
params.layout = NVBUF_LAYOUT_BLOCK_LINEAR;
params.colorFormat = NVBUF_COLOR_FORMAT_YUV420;
params.memtag = NvBufSurfaceTag_VIDEO_CONVERT;
// Allocate Nv buffer for YUV420 image
ret = NvBufSurf::NvAllocate(¶ms, 1, &dst_dma_fd);
/ Set Nv transform params for UYVY to YUV420
NvBufSurf::NvCommonTransformParams transform_params;
transform_params.src_top = 0;
transform_params.src_left = 0;
transform_params.src_width = width;
transform_params.src_height = height;
transform_params.dst_top = 0;
transform_params.dst_left = 0;
transform_params.dst_width = width;
transform_params.dst_height = height;
transform_params.flag = NVBUFSURF_TRANSFORM_FILTER;
transform_params.flip = NvBufSurfTransform_None;
// GPU-Lanzos, VIC-Smart interpolation
transform_params.filter = NvBufSurfTransformInter_Algo3;
// Transform UYVY to YUV420
int transform_res = NvBufSurf::NvTransform(&transform_params, src_dma_fd, dst_dma_fd);
// Write the JPEG image from the YUV420 Nv buffer to disk
for (int i = 0; i < iterator_num; ++i)
{
ret = jpegenc->encodeFromFd(dst_dma_fd, JCS_YCbCr, &out_buf_ptr, out_buf_size, config_->output_quality_);
if (ret < 0)
{
return absl::InternalError("Error while encoding from fd");
}
std::string output_filename = "my_jpeg_image.jpeg";
std::ofstream outFile(output_filename, std::ios::binary);
outFile.write(reinterpret_cast<char *>(out_buf_ptr), out_buf_size);
outFile.close();
}