Looking for details on how to perform hwaccel JPEG encoding on the Jetson Xavier running Jetpack 5.1.3. Currently using CUDA 11.4 and a source GpuMat with BRG colorspace.
The sample code in the Jetson Multimedia API provides samples for JPEG endode/decode but uses a file source rather than GPU memory source.
The question that I have is:
How to convert a GpuMat with a BGR colorspace image to a NvBufSurface so we can get a fd and use the encodeFromFd function from the Jetson Multimedia API and perform hwaccel JPEG encoding?
Hi,
Hardware JPEG encoder supports YUV420 format, so possible solution will be like:
Please refer to the sample for mapping NvBufSurface in RGBA to gpuMat:
Error generated while running the code after connecting the camera - #15 by DaveYYY
The following code fails with the call to NvBufSurfTransform returning error code -3, NvBufSurfTransformError_Invalid_Params.
NvBufSurface* nvbufSurfaceRGBA = nullptr;
NvBufSurface* nvbufSurfaceYUV = nullptr;
NvBufSurfaceCreateParams createParams;
createParams.gpuId = 0;
createParams.width = 1920;
createParams.height = 1080;
createParams.layout = NVBUF_LAYOUT_PITCH;
// Step 1: Allocate NvBufSurface in YUV420
createParams.colorFormat = NVBUF_COLOR_FORMAT_YUV420;
createParams.memType = NVBUF_MEM_CUDA_UNIFIED;
if (NvBufSurfaceCreate(&nvbufSurfaceYUV, 1, &createParams) != 0) {
std::cerr << "NvBufSurf::NvBufSurfaceCreate() FAILED, Failed to create YUV NvBufSurface" << std::endl;
return;
}
// Step 2: Allocate NvBufSurface in RGBA and map to gpuMat
createParams.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
if (NvBufSurfaceCreate(&nvbufSurfaceRGBA, 1, &createParams) != 0) {
std::cerr << "NvBufSurf::NvBufSurfaceCreate() FAILED, Failed to create RGBA NvBufSurface" << std::endl;
return;
}
// Step 3: Convert data from BGR gpuMat to the NvBufSurface in RGBA
cv::cuda::GpuMat gpuMatRGBA;
cv::cuda::cvtColor(gpuMatBGR, gpuMatRGBA, cv::COLOR_BGR2RGBA);
// Map RGBA NvBufSurface to GpuMat
NvBufSurfaceMap(nvbufSurfaceRGBA, 0, 0, NVBUF_MAP_WRITE);
NvBufSurfaceSyncForDevice(nvbufSurfaceRGBA, 0, 0);
cv::cuda::GpuMat nvGpuMatRGBA(createParams.height, createParams.width, CV_8UC4,
nvbufSurfaceRGBA->surfaceList[0].mappedAddr.addr[0],
nvbufSurfaceRGBA->surfaceList[0].pitch);
// Copy data from RGBA GpuMat to mapped NvBufSurface
gpuMatRGBA.copyTo(nvGpuMatRGBA);
NvBufSurfaceUnMap(nvbufSurfaceRGBA, 0, 0);
NvBufSurfTransformRect srcRect, dstRect;
srcRect.top = 0;
srcRect.left = 0;
srcRect.width = createParams.width;
srcRect.height = createParams.height;
dstRect.top = 0;
dstRect.left = 0;
dstRect.width = createParams.width;
dstRect.height = createParams.height;
NvBufSurfTransformConfigParams transformConfigParams;
transformConfigParams.gpu_id = 0;
transformConfigParams.compute_mode = NvBufSurfTransformCompute_Default;
NvBufSurfTransform_Error err = NvBufSurfTransformSetSessionParams(&transformConfigParams);
NvBufSurfTransformParams transformParams;
transformParams.src_rect = &srcRect;
transformParams.dst_rect = &dstRect;
transformParams.transform_flag = NVBUFSURF_TRANSFORM_FILTER;
transformParams.transform_flip = NvBufSurfTransform_None;
transformParams.transform_filter = NvBufSurfTransformInter_Default;
// Step 4: Call NvBufSurfTransform() to convert NvBufSurface in RGBA to NvBufSurface in YUV420
err = NvBufSurfTransform(nvbufSurfaceRGBA, nvbufSurfaceYUV, &transformParams);
if (err != NvBufSurfTransformError_Success) {
std::cerr << "NvBufSurfTransform failed with error " << err << std::endl;
return;
}
Is this the correct approach in preparation of calling the NvJPEGEncoder::encodeFromFd or NvJPEGEncoder::encodeFromBuffer method to perform hwaccel JPEG encoding?
Hi,
Please try
createParams.memType = NVBUF_MEM_SURFACE_ARRAY;
If the issue is still present, please try NvBufSurfaceAllocate(). There is reference code in
/usr/src/jetson_multimedia_api/samples/common/classes/NvBufSurface.cpp
After making the change to use:
createParams.memType = NVBUF_MEM_SURFACE_ARRAY;
The following message is shown:
nvbufsurface: Wrong buffer index (0)
After this call:
NvBufSurfaceSyncForDevice(nvbufSurfaceRGBA, 0, 0);
And this call:
gpuMatRGBA.copyTo(nvGpuMatRGBA);
Results in this error:
terminate called after throwing an instance of 'cv::Exception'
what(): OpenCV(4.5.5) /debout/openalpr-opencv_4.5.5.orig/modules/core/src/cuda/gpu_mat.cu:273: error: (-217:Gpu API call) invalid argument in function 'copyTo'
This is the code producing these errors:
NvBufSurface* nvbufSurfaceRGBA = nullptr;
NvBufSurface* nvbufSurfaceYUV = nullptr;
NvBufSurfaceCreateParams createParams;
createParams.gpuId = gpu_id;
createParams.width = in_width;
createParams.height = in_height;
createParams.layout = NVBUF_LAYOUT_PITCH;
// Step 1: Allocate NvBufSurface in YUV420
createParams.colorFormat = NVBUF_COLOR_FORMAT_YUV420;
createParams.memType = NVBUF_MEM_SURFACE_ARRAY;
if (NvBufSurfaceCreate(&nvbufSurfaceYUV, 1, &createParams) != 0) {
std::cerr << "NvBufSurf::NvBufSurfaceCreate() FAILED, Failed to create YUV NvBufSurface" << std::endl;
return;
}
// Step 2: Allocate NvBufSurface in RGBA and map to gpuMat
createParams.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
if (NvBufSurfaceCreate(&nvbufSurfaceRGBA, 1, &createParams) != 0) {
std::cerr << "NvBufSurf::NvBufSurfaceCreate() FAILED, Failed to create RGBA NvBufSurface" << std::endl;
return;
}
// Step 3: Convert data from BGR gpuMat to the NvBufSurface in RGBA
cv::cuda::GpuMat gpuMatRGBA;
cv::cuda::cvtColor(gpuMatBGR, gpuMatRGBA, cv::COLOR_BGR2RGBA);
// Map RGBA NvBufSurface to GpuMat
NvBufSurfaceMap(nvbufSurfaceRGBA, 0, 0, NVBUF_MAP_WRITE);
NvBufSurfaceSyncForDevice(nvbufSurfaceRGBA, 0, 0);
cv::cuda::GpuMat nvGpuMatRGBA(createParams.height, createParams.width, CV_8UC4,
nvbufSurfaceRGBA->surfaceList[0].mappedAddr.addr[0],
nvbufSurfaceRGBA->surfaceList[0].pitch);
// Copy data from RGBA GpuMat to mapped NvBufSurface
gpuMatRGBA.copyTo(nvGpuMatRGBA);
NvBufSurfaceUnMap(nvbufSurfaceRGBA, 0, 0);
NvBufSurfTransformRect srcRect, dstRect;
srcRect.top = 0;
srcRect.left = 0;
srcRect.width = createParams.width;
srcRect.height = createParams.height;
dstRect.top = 0;
dstRect.left = 0;
dstRect.width = createParams.width;
dstRect.height = createParams.height;
NvBufSurfTransformConfigParams transformConfigParams;
transformConfigParams.gpu_id = 0;
transformConfigParams.compute_mode = NvBufSurfTransformCompute_Default;
NvBufSurfTransform_Error err = NvBufSurfTransformSetSessionParams(&transformConfigParams);
NvBufSurfTransformParams transformParams;
transformParams.src_rect = &srcRect;
transformParams.dst_rect = &dstRect;
transformParams.transform_flag = NVBUFSURF_TRANSFORM_FILTER;
transformParams.transform_flip = NvBufSurfTransform_None;
transformParams.transform_filter = NvBufSurfTransformInter_Default;
// Step 4: Call NvBufSurfTransform() to convert NvBufSurface in RGBA to NvBufSurface in YUV420
err = NvBufSurfTransform(nvbufSurfaceRGBA, nvbufSurfaceYUV, &transformParams);
if (err != NvBufSurfTransformError_Success) {
std::cerr << "NvBufSurfTransform failed with error " << err << std::endl;
return;
}
Using NvBufSurfaceAllocate rather than NvBufSurfaceCreate results in the same errors.
Hi,
Please set up this after the NvBufSurface is created:
nvbuf_surf->numFilled = 1;
This has made progress but the following code results in a black JPEG image of approx 1.7KB:
int fd = nvbufSurfaceYUV->surfaceList[0].bufferDesc;
std::cout << "NvSurfEncoder::encode() - fd: " << fd << std::endl;
unsigned long out_buf_size = 0;
unsigned char* out_buf = nullptr;
ret = jpegenc->encodeFromFd(fd, JCS_YCbCr, &out_buf, out_buf_size, jpeg_quality);
if (ret < 0) {
std::cerr << "encodeFromFd() FAILED, Error while encoding from fd" << std::endl;
return;
}
std::cout << "NvSurfEncoder::encode() - out_buf_size: " << out_buf_size << std::endl;
Hi,
Your code does not allocate the output buffer. You may add
out_buf_size = ctx.in_width * ctx.in_height * 3 / 2;
out_buf = new unsigned char[out_buf_size];
And please refer to the sample:
/usr/src/jetson_multimedia_api/samples/05_jpeg_encode
Same result unfortunately…
Hi,
It looks like the data is not put into the NvBufSurface. Please call dump_dmabuf() to dump the surfaces to a file and check. YUV420 has 3 planes, so please call it for plane 0, 1, 2. There is reference code of dumping the surfaces in 00_video_decode.
out.zip (285 Bytes)
Does not appear to contain valid YUV data. Since the input was a GpuMat with BRG colorspace data converted to a NvBufSurface with RGBA data before transforming to a NvBufSurface with YUV data, what is the best way to validate the input and intermediate data buffers?
This is the current code implementation:
uint32_t in_width = gpuMatBGR.cols;
uint32_t in_height = gpuMatBGR.rows;
NvBufSurface *nvbufSurfaceRGBA = nullptr;
NvBufSurface *nvbufSurfaceYUV = nullptr;
NvBufSurfaceAllocateParams input_params = {{0}};
input_params.params.width = in_width;
input_params.params.height = in_height;
input_params.params.memType = NVBUF_MEM_SURFACE_ARRAY;
input_params.params.layout = NVBUF_LAYOUT_PITCH;
input_params.params.colorFormat = NVBUF_COLOR_FORMAT_YUV420;
input_params.memtag = NvBufSurfaceTag_VIDEO_CONVERT;
int ret = NvBufSurfaceAllocate(&nvbufSurfaceYUV, 1, &input_params);
if (ret != 0)
{
std::cerr << "NvBufSurf::NvBufSurfaceAllocate() FAILED, Failed to create YUV NvBufSurface" << std::endl;
return;
}
nvbufSurfaceYUV->numFilled = 1;
input_params.params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
ret = NvBufSurfaceAllocate(&nvbufSurfaceRGBA, 1, &input_params);
if (ret != 0)
{
std::cerr << "NvBufSurf::NvBufSurfaceAllocate() FAILED, Failed to create RGBA NvBufSurface" << std::endl;
return;
}
nvbufSurfaceRGBA->numFilled = 1;
// Step 3: Convert data from BGR gpuMat to the NvBufSurface in RGBA
cv::cuda::GpuMat gpuMatRGBA;
cv::cuda::cvtColor(gpuMatBGR, gpuMatRGBA, cv::COLOR_BGR2RGBA);
// Map RGBA NvBufSurface to GpuMat
NvBufSurfaceMap(nvbufSurfaceRGBA, 0, 0, NVBUF_MAP_WRITE);
NvBufSurfaceSyncForDevice(nvbufSurfaceRGBA, 0, 0);
cv::cuda::GpuMat nvGpuMatRGBA(in_width, in_height, CV_8UC4,
nvbufSurfaceRGBA->surfaceList[0].mappedAddr.addr[0],
nvbufSurfaceRGBA->surfaceList[0].pitch);
// Copy data from RGBA GpuMat to mapped NvBufSurface
gpuMatRGBA.copyTo(nvGpuMatRGBA);
NvBufSurfaceUnMap(nvbufSurfaceRGBA, 0, 0);
NvBufSurfTransformRect srcRect, dstRect;
srcRect.top = 0;
srcRect.left = 0;
srcRect.width = in_width;
srcRect.height = in_height;
dstRect.top = 0;
dstRect.left = 0;
dstRect.width = in_width;
dstRect.height = in_height;
NvBufSurfTransformConfigParams transformConfigParams;
transformConfigParams.gpu_id = 0;
transformConfigParams.compute_mode = NvBufSurfTransformCompute_Default;
NvBufSurfTransform_Error err = NvBufSurfTransformSetSessionParams(&transformConfigParams);
NvBufSurfTransformParams transformParams;
transformParams.src_rect = &srcRect;
transformParams.dst_rect = &dstRect;
transformParams.transform_flag = NVBUFSURF_TRANSFORM_FILTER;
transformParams.transform_flip = NvBufSurfTransform_None;
transformParams.transform_filter = NvBufSurfTransformInter_Nearest;
// Step 4: Call NvBufSurfTransform() to convert NvBufSurface in RGBA to NvBufSurface in YUV420
err = NvBufSurfTransform(nvbufSurfaceRGBA, nvbufSurfaceYUV, &transformParams);
if (err != NvBufSurfTransformError_Success)
{
std::cerr << "NvBufSurfTransform failed with error " << err << std::endl;
return;
}
// Step 5: Encode the NvBufSurface in YUV420
NvJPEGEncoder *jpegenc = NvJPEGEncoder::createJPEGEncoder("jpenenc");
if (!jpegenc)
{
std::cerr << "NvJPEGEncoder::createJPEGEncoder() FAILED" << std::endl;
return;
}
int fd = nvbufSurfaceYUV->surfaceList[0].bufferDesc;
unsigned long out_buf_size = in_width * in_height * 3 / 2;
unsigned char *out_buf = new unsigned char[out_buf_size];
ret = jpegenc->encodeFromFd(fd, JCS_YCbCr, &out_buf, out_buf_size, jpeg_quality);
if (ret < 0)
{
std::cerr << "encodeFromFd() FAILED, Error while encoding from fd" << std::endl;
return;
}
Hi,
Please dump nvbufSurfaceRGBA to file and check if it contains valid RGBA data.
And for mapping to gpuMat, please check
Error generated while running the code after connecting the camera - #15 by DaveYYY
Calling the CUDA functions is required. Please add the function calls and try.
In the sample code last provided, where does the required mapping to gpuMat need to occur? Are you referring to the gpuMatBGR being used as the input in the sample code?
With regards to the required CUDA functions, can you please elaborate what functions need to be called and where in the provided sample code.
Hi,
Please refer to the code:
{
NvBufSurfaceMapEglImage(surface, 0);
CUresult status;
CUeglFrame eglFrame;
CUgraphicsResource pResource = NULL;
cudaFree(0);
status = cuGraphicsEGLRegisterImage(&pResource,
surface->surfaceList[0].mappedAddr.eglImage,
CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE);
if (status != CUDA_SUCCESS)
{
printf("cuGraphicsEGLRegisterImage failed: %d \n",status);
}
status = cuGraphicsResourceGetMappedEglFrame(&eglFrame, pResource, 0, 0);
status = cuCtxSynchronize();
if (create_filter) {
filter = cv::cuda::createSobelFilter(CV_8UC4, CV_8UC4, 1, 0, 3, 1, cv::BORDER_DEFAULT);
//filter = cv::cuda::createGaussianFilter(CV_8UC4, CV_8UC4, cv::Size(31,31), 0, 0, cv::BORDER_DEFAULT);
create_filter = false;
}
cv::cuda::GpuMat d_mat(h, w, CV_8UC4, eglFrame.frame.pPitch[0]);
filter->apply(d_mat, d_mat);
status = cuCtxSynchronize();
status = cuGraphicsUnregisterResource(pResource);
NvBufSurfaceUnMapEglImage(surface, 0);
}
I saw the code you referenced in the other post, but where does this apply to the sample code provided? Where does the EglImage apply? Is it the cudaFree and cuCtxSynchronize calls that need to be applied?
The questions I posted were directed towards another post rather than being answered directly.
Hi,
Please try
{
NvBufSurfaceMapEglImage(surface, 0);
CUresult status;
CUeglFrame eglFrame;
CUgraphicsResource pResource = NULL;
cudaFree(0);
status = cuGraphicsEGLRegisterImage(&pResource,
surface->surfaceList[0].mappedAddr.eglImage,
CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE);
if (status != CUDA_SUCCESS)
{
printf("cuGraphicsEGLRegisterImage failed: %d \n",status);
}
status = cuGraphicsResourceGetMappedEglFrame(&eglFrame, pResource, 0, 0);
status = cuCtxSynchronize();
cv::cuda::GpuMat d_mat(h, w, CV_8UC4, eglFrame.frame.pPitch[0]);
gpuMatRGBA.copyTo(d_mat);
status = cuCtxSynchronize();
status = cuGraphicsUnregisterResource(pResource);
NvBufSurfaceUnMapEglImage(surface, 0);
}
And see if gpuMatRGBA can be successfully copied to d_mat with the code.
With the current implementation in the attached file, the NvBufSurfaceMapEglImage call results in:
nvbufsurface: Failed to create EGLImage.
NvSurfEncoder.zip (2.8 KB)
Hi,
Please share the Make command and steps so that we can build and run the sample. The cpp file seems partial since nowhere calls testEncode(). Would need your help to share a full test sample and steps.
jpeg-hwaccel.zip (86.1 KB)
opencv-deb.zip (28.1 MB)
Dependencies:
To build:
mkdir build && cd build
cmake …
make
Usage:
./test_jpeg </path/to/input.jpg> </path/to/output.jpg>