Background subtraction works perfectly on CPU backend but does nothing on CUDA using VPI 2.0

Environment:

Device: Jetson NX
JetPack: 4.6
OpenCV: 4.4.0
GStreamer: 1.16.2
Deepstream: 6.1
CUDA: 10.2

Problem:

Background subtraction works perfectly on CPU backend but does nothing on CUDA.
Reproduce with a one-line change on line 131: change CUDA to CPU.
(Note: if you have Clang’s -Wunused on, you may have to disable it, as CUDA won’t be used.)

Code

// vpi_check.h

#define VPI_CHECK(STMT)                                   \
  do {                                                    \
    VPIStatus status = (STMT);                            \
    if (status != VPI_SUCCESS) {                          \
      char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH];         \
      vpiGetLastStatusMessage(buffer, sizeof(buffer));    \
      std::ostringstream ss;                              \
      ss << vpiStatusGetName(status) << ": \"" << buffer  \
         << "\" from `" << #STMT << '`';                  \
      throw std::runtime_error(ss.str());                 \
    }                                                     \
  } while (0)



//%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



// vpi_raii.h

#include <vpi/OpenCVInterop.hpp>

extern "C" {
#include <vpi/algo/BackgroundSubtractor.h>
#include <vpi/Image.h>
#include <vpi/Stream.h>
}

#include <opencv2/imgcodecs.hpp>

namespace vpi {

// Forward declarations, since the class macro confuses an IDE parser
template <uint64_t backend> class Stream;
template <uint64_t backend, int32_t w, int32_t h, VPIImageFormat format> class Image;



#define VPI_CLASS(Name, ...)                              \
class Name {                                              \
 protected:                                               \
  VPI##Name internal;                                     \
 public:                                                  \
  Name() { vpi##Name##Create(__VA_ARGS__, &internal); }   \
  ~Name() { vpi##Name##Destroy(internal); }               \
  operator VPI##Name() const { return internal; }
// Class does NOT close itself! Define your own methods then close it with };



template <uint64_t backend>
VPI_CLASS(Stream, backend)
  void Sync() const { vpiStreamSync(internal); }
};



template <uint64_t backend, int32_t w, int32_t h, VPIImageFormat format>
VPI_CLASS(Image, w, h, format, backend)
  Image(const cv::Mat& mat) {
    VPI_CHECK(vpiImageCreateWrapperOpenCVMat(mat, format, backend, &internal));
    if ((mat.cols != w) || (mat.rows != h)) {
      throw std::runtime_error{
            "Creating a (" + std::to_string(w) + ", " + std::to_string(h) + ") ImageWrapper "
            "from a (" + std::to_string(mat.cols) + ", " + std::to_string(mat.rows) + ") cv::Mat"};
    }
  }
  void Save(const std::string& path) const {
    VPIImageData px;
    VPIStatus response;
    do { // Wait to acquire the lock
      response = vpiImageLockData(this->internal, VPI_LOCK_READ,
            VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR, &px);
      if (response == VPI_SUCCESS) { break; }
      else if (response != VPI_ERROR_BUFFER_LOCKED) { VPI_CHECK(response); }
    } while (true);
    VPIImagePlanePitchLinearRec& plane = px.buffer.pitch.planes[0];
    int cv_type = CV_MAKETYPE(CV_8U, vpiImageFormatGetChannelCount(format));
    cv::Mat mat_out{plane.height, plane.width, cv_type,
          plane.data, static_cast<size_t>(plane.pitchBytes)};
    cv::imwrite(path, mat_out);
    VPI_CHECK(vpiImageUnlock(this->internal));
  }
};

} // namespace vpi



//%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



// main.cc

#include <opencv2/videoio.hpp>

#include <string>



static constexpr uint64_t CUDA{VPI_BACKEND_CUDA};
static constexpr uint64_t CPU{VPI_BACKEND_CPU};

static constexpr VPIImageFormat RGB8{VPI_IMAGE_FORMAT_RGB8};

static constexpr VPIBackgroundSubtractorParams bsub_params{
  .varThreshold = 16,
  .detectShadow = true,
  .shadowPixelValue = 127,
  .learningRate = 0.001f
};



template <uint64_t backend, int32_t w, int32_t h, VPIImageFormat format>
class Payload {
 protected:
  VPIPayload internal;
 public:
  Payload() { VPI_CHECK(vpiCreateBackgroundSubtractor(backend, w, h, format, &internal)); }
  ~Payload() { vpiPayloadDestroy(internal); }
  operator VPIPayload() const { return internal; }
};



static constexpr uint64_t kBackend = CUDA;
static constexpr VPIImageFormat kFormat = RGB8;
static constexpr int32_t kW = 1280;
static constexpr int32_t kH = 720;

int main() {
  static constexpr const char* video_path_raw{"../motion_detection/9.mp4"};
  const std::string video_path{video_path_raw};
  cv::VideoCapture vc{video_path};
  if (!vc.isOpened()) { throw std::runtime_error{"Couldn't open " + video_path}; }
  cv::Mat frame;
  if (!vc.read(frame)) { throw std::runtime_error{"No video data found in " + video_path}; }
  vpi::Stream<kBackend> stream{};
  vpi::Image<kBackend, kW, kH, kFormat> im{frame};
  Payload<kBackend, kW, kH, kFormat> payload{};
  vpi::Image<kBackend | CPU, kW, kH, VPI_IMAGE_FORMAT_U8> out_mask{};
  vpi::Image<kBackend | CPU, kW, kH, kFormat> out_im{};
  uint16_t i{0};
  do {
    VPI_CHECK(vpiSubmitBackgroundSubtractor(stream, kBackend, payload, im, out_mask, out_im, &bsub_params));
    stream.Sync();
    out_mask.Save("mask_" + std::to_string(i) + ".png");
    out_im.Save("im_" + std::to_string(i) + ".png");
    ++i;
  } while (vc.read(frame));
}

Moved to the Jetpack forum

I’m closing this topic due to there is no update from you for a period, assuming this issue was resolved.
If still need the support, please open a new topic. Thanks

Hi,

Just want to confirm your environment first.
Do you use JetPack 5.0.2 (VPI 2.1) or JetPack 4.6.2 (VPI 1.2)?

Thanks.