I am trying to run VPI examples and see the output image over the SSH (with x11 forwarding).
My platform details:
- Orin NX Module with Connect Tech Hadron Carrier Board
- Jetpack 5.1.2, L4T 35.4.1
- OpenCV with CUDA 4.9.0
- VPI 2.3.9
- CUDA 11.4.315
I have tried both SSH connection methods. I do not connect any monitor, there is no display output on the carrier board.
1) Without -X option
OpenCV function cv::imshow() doesn’t work but VPI example can run, but no output image is shown. Output can be stored on the disk by cv::imwrite().
2) With -X option
OpenCV function cv::imshow() works without any problem (For OpenCV only examples, not with VPI), I can see the output image at runtime. But with -X option I get this error;
libEGL warning: DRI3: failed to query the version
libEGL warning: DRI2: failed to authenticate
VPI_ERROR_INTERNAL: VIC|NVENC|OFA backend(s) error: (EGL_BAD_DISPLAY)
I am trying with the test code below. I take this example code from official VPI SDK.
#include <opencv2/core/version.hpp>
#include <opencv2/opencv.hpp>
#if CV_MAJOR_VERSION >= 3
# include <opencv2/imgcodecs.hpp>
#else
# include <opencv2/highgui/highgui.hpp>
#endif
#include <vpi/OpenCVInterop.hpp>
#include <vpi/Image.h>
#include <vpi/Status.h>
#include <vpi/Stream.h>
#include <vpi/algo/ConvertImageFormat.h>
#include <vpi/algo/Rescale.h>
#include <cassert>
#include <cstring> // for memset
#include <iostream>
#include <sstream>
#define CHECK_STATUS(STMT) \
do \
{ \
VPIStatus status = (STMT); \
if (status != VPI_SUCCESS) \
{ \
char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH]; \
vpiGetLastStatusMessage(buffer, sizeof(buffer)); \
std::ostringstream ss; \
ss << vpiStatusGetName(status) << ": " << buffer; \
throw std::runtime_error(ss.str()); \
} \
} while (0);
int main(int argc, char *argv[])
{
// OpenCV image that will be wrapped by a VPIImage.
// Define it here so that it's destroyed *after* wrapper is destroyed
cv::Mat cvImage;
// VPI objects that will be used
VPIImage image = NULL;
VPIImage imageNV12 = NULL;
VPIImage outputNV12 = NULL;
VPIImage output = NULL;
VPIStream stream = NULL;
int retval = 0;
try
{
if (argc != 3)
{
throw std::runtime_error(std::string("Usage: ") + argv[0] + " <cpu|vic|cuda> <input image>");
}
std::string strBackend = argv[1];
std::string strInputFileName = argv[2];
// Load the input image
cvImage = cv::imread(strInputFileName);
if (cvImage.empty())
{
throw std::runtime_error("Can't open '" + strInputFileName + "'");
}
assert(cvImage.type() == CV_8UC3);
// Now parse the backend
VPIBackend backend;
if (strBackend == "cpu")
{
backend = VPI_BACKEND_CPU;
}
else if (strBackend == "cuda")
{
backend = VPI_BACKEND_CUDA;
}
else if (strBackend == "vic")
{
backend = VPI_BACKEND_VIC;
}
else
{
throw std::runtime_error("Backend '" + strBackend + "' not recognized, it must be either cpu, cuda or vic");
}
// 1. Initialization phase ---------------------------------------
// Create the stream for the given backend. We'll also enable CUDA for gaussian filter.
CHECK_STATUS(vpiStreamCreate(backend | VPI_BACKEND_CUDA, &stream));
// We now wrap the loaded image into a VPIImage object to be used by VPI.
// VPI won't make a copy of it, so the original
// image must be in scope at all times.
CHECK_STATUS(vpiImageCreateWrapperOpenCVMat(cvImage, 0, &image));
// Create a temporary image to hold the input converted to NV12.
CHECK_STATUS(vpiImageCreate(cvImage.cols, cvImage.rows, VPI_IMAGE_FORMAT_NV12_ER, 0, &imageNV12));
// Now create the output image.
CHECK_STATUS(vpiImageCreate(cvImage.cols / 2, cvImage.rows / 3, VPI_IMAGE_FORMAT_NV12_ER, 0, &outputNV12));
// And the output image converted back to BGR8
CHECK_STATUS(vpiImageCreate(cvImage.cols / 2, cvImage.rows / 3, VPI_IMAGE_FORMAT_BGR8, 0, &output));
// 2. Computation phase ---------------------------------------
// Convert input from BGR8 to NV12
CHECK_STATUS(vpiSubmitConvertImageFormat(stream, VPI_BACKEND_CUDA, image, imageNV12, NULL));
// Now we downsample
CHECK_STATUS(vpiSubmitRescale(stream, backend, imageNV12, outputNV12, VPI_INTERP_LINEAR, VPI_BORDER_CLAMP, 0));
// Finally, convert the result back to BGR8
CHECK_STATUS(vpiSubmitConvertImageFormat(stream, VPI_BACKEND_CUDA, outputNV12, output, NULL));
// Wait until the algorithm finishes processing
CHECK_STATUS(vpiStreamSync(stream));
//putenv("DISPLAY=localhost:10.0"); // DISPLAY variable is unset by default, after this line imshow can work.
// Now let's retrieve the output image contents and output it to disk
{
// Lock output image to retrieve its data on cpu memory
VPIImageData outData;
CHECK_STATUS(vpiImageLockData(output, VPI_LOCK_READ, VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR, &outData));
// Returned data consists of host-accessible memory buffers in pitch-linear layout.
assert(outData.bufferType == VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR);
VPIImageBufferPitchLinear &outDataPitch = outData.buffer.pitch;
cv::Mat cvOut(outDataPitch.planes[0].height, outDataPitch.planes[0].width, CV_8UC3,
outDataPitch.planes[0].data, outDataPitch.planes[0].pitchBytes);
cv::imwrite("scaled_" + strBackend + ".png", cvOut);
cv::imshow("test", cvOut);
while(true)
if(cv::waitKey() == 27)
break;
// Done handling output image, don't forget to unlock it.
CHECK_STATUS(vpiImageUnlock(output));
}
}
catch (std::exception &e)
{
std::cerr << e.what() << std::endl;
retval = 1;
}
// Clean up
// Make sure stream is synchronized before destroying the objects
// that might still be in use.
vpiStreamSync(stream);
vpiImageDestroy(image);
vpiImageDestroy(imageNV12);
vpiImageDestroy(output);
vpiStreamDestroy(stream);
return retval;
}
To overcome this, I can modify the environment variable DISPLAY at the runtime by
putenv("DISPLAY=localhost:10.0").
The DISPLAY environment variable remains unset until the line where I call the imshow function, and thus the VPI library does not crash. To show the result image with imshow, the environment variable is changed with putenv and the result can be shown via SSH. There is a conflict here. While imshow can work with X11 routing, VPI does not. Conversely, VPI can also work (but the result image is not displayed).
However, I think this solution does not make sense.
- I think I should be able to see the result image produced with VPI over SSH, even without the
imshowfunction. Is this true? - What causes the error? When I couldn’t do this, I tried VNC (x11vnc, vino-server and etc). I connected to the remote device but there is only Nvidia logo on the screen (that wee see on boot sequence).
I have also tried adding “fake monitor”. Here is my xorg.conf file;
# Copyright (c) 2011-2013 NVIDIA CORPORATION. All Rights Reserved.
#
# This is the minimal configuration necessary to use the Tegra driver.
# Please refer to the xorg.conf man page for more configuration
# options provided by the X server, including display-related options
# provided by RandR 1.2 and higher.
# Disable extensions not useful on Tegra.
Section "Module"
Disable "dri"
SubSection "extmod"
Option "omit xfree86-dga"
EndSubSection
EndSection
Section "Device"
Identifier "Tegra0"
Driver "nvidia"
# Allow X server to be started even if no display devices are connected.
Option "AllowEmptyInitialConfiguration" "true"
EndSection
Section "Monitor"
Identifier "Configured Monitor"
HorizSync 31.5-48.5
VertRefresh 50-70
EndSection
Section "Screen"
Identifier "Default Screen"
Monitor "Configured Monitor"
Device "Tegra0"
DefaultDepth 24
SubSection "Display"
Depth 24
Modes "1280x720"
EndSubSection
EndSection
And my xrandr outputs both with -X and without -X ssh connection:
With -X:
Screen 0: minimum 320 x 200, current 1920 x 1080, maximum 16384 x 16384
eDP-1 connected primary 1920x1080+0+0 (normal left inverted right x axis y axis) 521mm x 293mm
1920x1080 60.00*+ 59.93
1680x1050 59.95 59.88
1600x1024 60.17
1400x1050 59.98
1600x900 59.95 59.82
1280x1024 60.02
1440x900 59.89
1400x900 59.96 59.88
1280x960 60.00
1440x810 59.97
1368x768 59.88 59.85
1360x768 59.80 59.96
1280x800 59.97 59.81 59.91
1152x864 60.00
1280x720 60.00 59.99 59.86 59.74
1024x768 60.04 60.00
960x720 60.00
928x696 60.05
896x672 60.01
1024x576 59.95 59.96 59.90 59.82
960x600 59.93 60.00
960x540 59.96 59.99 59.63 59.82
800x600 60.00 60.32 56.25
840x525 60.01 59.88
864x486 59.92 59.57
800x512 60.17
700x525 59.98
800x450 59.95 59.82
640x512 60.02
720x450 59.89
700x450 59.96 59.88
640x480 60.00 59.94
720x405 59.51 58.99
684x384 59.88 59.85
680x384 59.80 59.96
640x400 59.88 59.98
576x432 60.06
640x360 59.86 59.83 59.84 59.32
512x384 60.00
512x288 60.00 59.92
480x270 59.63 59.82
400x300 60.32 56.34
432x243 59.92 59.57
320x240 60.05
360x202 59.51 59.13
320x180 59.84 59.32
DP-1 disconnected (normal left inverted right x axis y axis)
HDMI-1 disconnected (normal left inverted right x axis y axis)
Without -X:
xrandr --display :0
No protocol specified
Can't open display :0
xrandr --display :1
Can't open display :1