Environment:
Device: Jetson NX
JetPack: 4.6
OpenCV: 4.4.0
GStreamer: 1.16.2
Deepstream: 6.1
CUDA: 10.2
Problem:
Background subtraction works perfectly on CPU backend but does nothing on CUDA.
Reproduce with a one-line change on line 131: change CUDA
to CPU
.
(Note: if you have Clang’s -Wunused on, you may have to disable it, as CUDA
won’t be used.)
Code
// vpi_check.h
#define VPI_CHECK(STMT) \
do { \
VPIStatus status = (STMT); \
if (status != VPI_SUCCESS) { \
char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH]; \
vpiGetLastStatusMessage(buffer, sizeof(buffer)); \
std::ostringstream ss; \
ss << vpiStatusGetName(status) << ": \"" << buffer \
<< "\" from `" << #STMT << '`'; \
throw std::runtime_error(ss.str()); \
} \
} while (0)
//%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
// vpi_raii.h
#include <vpi/OpenCVInterop.hpp>
extern "C" {
#include <vpi/algo/BackgroundSubtractor.h>
#include <vpi/Image.h>
#include <vpi/Stream.h>
}
#include <opencv2/imgcodecs.hpp>
namespace vpi {
// Forward declarations, since the class macro confuses an IDE parser
template <uint64_t backend> class Stream;
template <uint64_t backend, int32_t w, int32_t h, VPIImageFormat format> class Image;
#define VPI_CLASS(Name, ...) \
class Name { \
protected: \
VPI##Name internal; \
public: \
Name() { vpi##Name##Create(__VA_ARGS__, &internal); } \
~Name() { vpi##Name##Destroy(internal); } \
operator VPI##Name() const { return internal; }
// Class does NOT close itself! Define your own methods then close it with };
template <uint64_t backend>
VPI_CLASS(Stream, backend)
void Sync() const { vpiStreamSync(internal); }
};
template <uint64_t backend, int32_t w, int32_t h, VPIImageFormat format>
VPI_CLASS(Image, w, h, format, backend)
Image(const cv::Mat& mat) {
VPI_CHECK(vpiImageCreateWrapperOpenCVMat(mat, format, backend, &internal));
if ((mat.cols != w) || (mat.rows != h)) {
throw std::runtime_error{
"Creating a (" + std::to_string(w) + ", " + std::to_string(h) + ") ImageWrapper "
"from a (" + std::to_string(mat.cols) + ", " + std::to_string(mat.rows) + ") cv::Mat"};
}
}
void Save(const std::string& path) const {
VPIImageData px;
VPIStatus response;
do { // Wait to acquire the lock
response = vpiImageLockData(this->internal, VPI_LOCK_READ,
VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR, &px);
if (response == VPI_SUCCESS) { break; }
else if (response != VPI_ERROR_BUFFER_LOCKED) { VPI_CHECK(response); }
} while (true);
VPIImagePlanePitchLinearRec& plane = px.buffer.pitch.planes[0];
int cv_type = CV_MAKETYPE(CV_8U, vpiImageFormatGetChannelCount(format));
cv::Mat mat_out{plane.height, plane.width, cv_type,
plane.data, static_cast<size_t>(plane.pitchBytes)};
cv::imwrite(path, mat_out);
VPI_CHECK(vpiImageUnlock(this->internal));
}
};
} // namespace vpi
//%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
// main.cc
#include <opencv2/videoio.hpp>
#include <string>
static constexpr uint64_t CUDA{VPI_BACKEND_CUDA};
static constexpr uint64_t CPU{VPI_BACKEND_CPU};
static constexpr VPIImageFormat RGB8{VPI_IMAGE_FORMAT_RGB8};
static constexpr VPIBackgroundSubtractorParams bsub_params{
.varThreshold = 16,
.detectShadow = true,
.shadowPixelValue = 127,
.learningRate = 0.001f
};
template <uint64_t backend, int32_t w, int32_t h, VPIImageFormat format>
class Payload {
protected:
VPIPayload internal;
public:
Payload() { VPI_CHECK(vpiCreateBackgroundSubtractor(backend, w, h, format, &internal)); }
~Payload() { vpiPayloadDestroy(internal); }
operator VPIPayload() const { return internal; }
};
static constexpr uint64_t kBackend = CUDA;
static constexpr VPIImageFormat kFormat = RGB8;
static constexpr int32_t kW = 1280;
static constexpr int32_t kH = 720;
int main() {
static constexpr const char* video_path_raw{"../motion_detection/9.mp4"};
const std::string video_path{video_path_raw};
cv::VideoCapture vc{video_path};
if (!vc.isOpened()) { throw std::runtime_error{"Couldn't open " + video_path}; }
cv::Mat frame;
if (!vc.read(frame)) { throw std::runtime_error{"No video data found in " + video_path}; }
vpi::Stream<kBackend> stream{};
vpi::Image<kBackend, kW, kH, kFormat> im{frame};
Payload<kBackend, kW, kH, kFormat> payload{};
vpi::Image<kBackend | CPU, kW, kH, VPI_IMAGE_FORMAT_U8> out_mask{};
vpi::Image<kBackend | CPU, kW, kH, kFormat> out_im{};
uint16_t i{0};
do {
VPI_CHECK(vpiSubmitBackgroundSubtractor(stream, kBackend, payload, im, out_mask, out_im, &bsub_params));
stream.Sync();
out_mask.Save("mask_" + std::to_string(i) + ".png");
out_im.Save("im_" + std::to_string(i) + ".png");
++i;
} while (vc.read(frame));
}