Hi,
I created a small program where I use VPI to resize the cropped frames that are obtained from an IMX 219 camera (Raspberry Pi Camera V2) using GStreamer in Jetson Nano, and visualizing the stream in my laptop using UDP pipeline also with GStreamer. It works well with an output frame size of 1280 x 720, but if I change the resized output width to something else (modify macro OUTPUT_W), I obtain a noisy image as the one attached to this topic. However, I don’t see this issue if I only modify the height.
I have tried with a clean install of JetPack 4.5 and 4.5.1 to include VPI Production Release 1.0, and with a USB camera, but the results are the same. Also, if I replace VPI’s resizing with resize in OpenCV (CPU version), the issue doesn’t appear there.
Perhaps I missed something from VPI’s documentation, please let me know.
Any help is very well appreciated.
Please see my code below (update the HOST macro to match your IP):
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
#include <vpi/OpenCVInterop.hpp>
#include <vpi/Image.h>
#include <vpi/Status.h>
#include <vpi/Stream.h>
#include <vpi/CUDAInterop.h>
#include <vpi/algo/ConvertImageFormat.h>
#include <vpi/algo/Rescale.h>
#include <cstring>
#include <iostream>
#include <sstream>
#include <thread>
#define HOST_IP "XXX.XXX.X.XX"
#define GST_SRC_PIPELINE_USB \
"v4l2src device=/dev/video1 " \
"! image/jpeg, width=1280, height=720, framerate=30/1 " \
"! jpegdec ! videoconvert ! video/x-raw, format=(string)BGR ! appsink"
#define GST_SRC_PIPELINE_IMX \
"nvarguscamerasrc " \
"! video/x-raw(memory:NVMM), width=(int)1280, height=(int)720, " \
"format=(string)NV12, framerate=(fraction)60/1 " \
"! nvvidconv flip-method=0 " \
"! video/x-raw, width=(int)1280, height=(int)720, format=(string)BGRx " \
"! videoconvert ! video/x-raw, format=(string)BGR ! appsink"
#define GST_SINK_PIPELINE \
"appsrc ! videoconvert ! x264enc speed-preset=superfast tune=zerolatency " \
"! rtph264pay ! udpsink port=3445 host=" HOST_IP
#define OUTVID_FPS 30.0
#define NUM_WINDOWS 4
#define OUTPUT_W 700
#define OUTPUT_H 400
#define CHECK_STATUS(STMT) \
do \
{ \
VPIStatus status = (STMT); \
if (status != VPI_SUCCESS) \
{ \
char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH]; \
vpiGetLastStatusMessage(buffer, sizeof(buffer)); \
std::ostringstream ss; \
ss << vpiStatusGetName(status) << ": " << buffer; \
throw std::runtime_error(ss.str()); \
} \
} while (0);
static bool keep_running = true;
static VPIImage
ToVPIImage(VPIImage image, const cv::Mat &frame)
{
if (image == nullptr)
{
// Ceate a VPIImage that wraps the frame
CHECK_STATUS(vpiImageCreateOpenCVMatWrapper(frame, 0, &image));
}
else
{
// reuse existing VPIImage wrapper to wrap the new frame.
CHECK_STATUS(vpiImageSetWrappedOpenCVMat(image, frame));
}
return image;
}
static void
userInput_thread(void)
{
while(keep_running) {
if (std::cin.get() == '\n')
{
keep_running = false;
}
}
}
int
main(int argc, char *argv[])
{
int retval = 0;
bool bypass_windows = false;
bool use_usb = false;
cv::VideoCapture invid;
cv::VideoWriter outvid;
cv::Mat srcimg;
cv::Mat crpimg;
cv::Mat sinkimg;
cv::Rect windows[NUM_WINDOWS];
VPIBackend backend = VPI_BACKEND_CUDA;
VPIStream stream;
VPIImage vpi_in_img = NULL;
VPIImage vpi_in_nv12_img = NULL;
VPIImage vpi_rsz_img = NULL;
VPIImage vpi_out_img = NULL;
VPIImageData vpi_out_data;
int w;
int h;
int fourcc;
int wincnt = 0;
int framecnt = 0;
std::thread inthread(&userInput_thread);
std::vector<std::string> args(argv, argv+argc);
for (size_t i = 1; i < args.size(); ++i) {
if ((args[i] == "-b") || (args[i] == "--bypass")) {
bypass_windows = true;
} else if ((args[i] == "-u") || (args[i] == "--usb")) {
use_usb = true;
}
}
if(use_usb)
{
invid.open(GST_SRC_PIPELINE_USB, cv::CAP_GSTREAMER);
}
else
{
invid.open(GST_SRC_PIPELINE_IMX, cv::CAP_GSTREAMER);
}
w = invid.get(cv::CAP_PROP_FRAME_WIDTH);
h = invid.get(cv::CAP_PROP_FRAME_HEIGHT);
fourcc = cv::VideoWriter::fourcc('H','2','6','4');
outvid.open(GST_SINK_PIPELINE, cv::CAP_GSTREAMER, fourcc, OUTVID_FPS,
cv::Size(OUTPUT_W, OUTPUT_H));
windows[0] = cv::Rect(0, 0, w / 2, h / 2);
windows[1] = cv::Rect((w / 2) - 1, 0, w / 2, h / 2);
windows[2] = cv::Rect(0, (h / 2) - 1, w / 2, h / 2);
windows[3] = cv::Rect((w / 2) - 1, (h / 2) - 1, w / 2, h / 2);
try
{
if(!bypass_windows)
{
CHECK_STATUS(vpiStreamCreate(backend, &stream));
// Create a temporary image to hold the input converted to NV12.
CHECK_STATUS(vpiImageCreate(w / 2, h / 2, VPI_IMAGE_FORMAT_NV12_ER,
0, &vpi_in_nv12_img));
// Now create the resized image.
CHECK_STATUS(vpiImageCreate(OUTPUT_W, OUTPUT_H,
VPI_IMAGE_FORMAT_NV12_ER, 0,
&vpi_rsz_img));
// And the resized image converted back to BGR8
CHECK_STATUS(vpiImageCreate(OUTPUT_W, OUTPUT_H,
VPI_IMAGE_FORMAT_BGR8, 0,
&vpi_out_img));
}
while(keep_running)
{
if(!invid.read(srcimg))
{
break;
}
if(!bypass_windows)
{
crpimg = srcimg(windows[wincnt]);
// We now wrap the loaded image into a VPIImage object to be
// used by VPI. VPI won't make a copy of it, so the original
// image must be in scope at all times.
vpi_in_img = ToVPIImage(vpi_in_img, crpimg);
// Convert input from BGR8 to NV12
CHECK_STATUS(vpiSubmitConvertImageFormat(stream, backend,
vpi_in_img,
vpi_in_nv12_img,
NULL));
// Now we downsample
CHECK_STATUS(vpiSubmitRescale(stream, backend, vpi_in_nv12_img,
vpi_rsz_img, VPI_INTERP_LINEAR,
VPI_BORDER_CLAMP, 0));
// Finally, convert the result back to BGR8
CHECK_STATUS(vpiSubmitConvertImageFormat(stream, backend,
vpi_rsz_img,
vpi_out_img, NULL));
CHECK_STATUS(vpiStreamSync(stream));
CHECK_STATUS(vpiImageLock(vpi_out_img, VPI_LOCK_READ,
&vpi_out_data));
CHECK_STATUS(vpiImageDataExportOpenCVMat(vpi_out_data,
&sinkimg));
outvid.write(sinkimg);
CHECK_STATUS(vpiImageUnlock(vpi_out_img));
framecnt++;
if(framecnt == OUTVID_FPS)
{
framecnt = 0;
wincnt++;
wincnt %= NUM_WINDOWS;
}
}
else
{
outvid.write(srcimg);
}
}
}
catch (std::exception &e)
{
std::cerr << "Exception found: " << e.what() << std::endl;
retval = 1;
}
inthread.join();
if(!bypass_windows)
{
if (stream != NULL)
{
vpiStreamSync(stream);
}
vpiImageDestroy(vpi_in_img);
vpiImageDestroy(vpi_in_nv12_img);
vpiImageDestroy(vpi_rsz_img);
vpiImageDestroy(vpi_out_img);
vpiStreamDestroy(stream);
}
return retval;
}
The CMakeList.txt to compile the code:
cmake_minimum_required(VERSION 3.5)
project(resize_windows_vpi)
find_package(vpi 1.0 REQUIRED)
find_package(OpenCV REQUIRED)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
add_executable(${PROJECT_NAME} main.cpp)
include_directories(/usr/local/cuda-10.2/targets/aarch64-linux/include)
target_link_libraries(${PROJECT_NAME} vpi)
target_link_libraries(${PROJECT_NAME} opencv_core)
target_link_libraries(${PROJECT_NAME} opencv_videoio)
The pipeline that I use to see the stream in my laptop (if you change the output width or height in main.cpp, don’t forget to do the same here to match the same values):
gst-launch-1.0 udpsrc port=3445 ! application/x-rtp ! rtph264depay ! queue \
! avdec_h264 ! videoconvert \
! "video/x-raw, width=1280, height=720, framerate=(fraction)30/1" ! ximagesink
