Use gstreamer pipeline to init opencv's videocapture, has memory leak when Repeatedly reset videocapture

environment:

  • Jetson Xavier NX
  • Jetpack 4.6
  • NVIDIA® Jetson™ Linux Driver Package (L4T) 32.6.1
  • gstreamer1.14.5
  • opencv4.1.1

code:

int CMyServer::open_camera() {
   string zstr = "rtspsrc location=rtsp://........ ! rtph264depay ! h264parse ! nvv4l2decoder ! nvvidconv ! video/x-raw, width=(int)1920, height=(int)1080, format=(string)BGRx ! videoconvert ! appsink";
   if (!video_cap_) {
       try {
//         video_cap_ = make_unique<VideoCapture>(zstr, cv::CAP_GSTREAMER);
           video_cap_.reset(new VideoCapture(zstr, cv::CAP_GSTREAMER));
           LOGF(INFO, "\n      reset video_cap_");
       } catch (...) {
           cerr << "error in opening camera\n" << flush;
           LOGF(WARNING, "\n      error in reseting video_cap_ first time");
       }
   }
   int wnum = 10;
   while (!video_cap_->isOpened() && wnum >= 0) {
       try {
           wnum--;
           video_cap_->open(zstr, cv::CAP_GSTREAMER);
       } catch (...) {
           cerr << "error in opening camera\n" << flush;
           LOGF(WARNING, "\n      error in reseting video_cap_ in while() and send camera err msg to client");
           break;
       }
       boost::this_thread::sleep(boost::posix_time::millisec(500));
   }
   if(!video_cap_->isOpened()||!video_cap_){
       return 1;
   }
   int w = static_cast<int>(video_cap_->get(CAP_PROP_FRAME_WIDTH));
   int h = static_cast<int>(video_cap_->get(CAP_PROP_FRAME_HEIGHT));
   Size videoSize(1920, 1080);
   if (!video_writer_) {
       video_writer_.reset(new VideoWriter("..//data//testgst-zhi-1920-30.avi",
                                                VideoWriter::fourcc('M', 'J', 'P', 'G'), 30,
                                               videoSize));
       LOGF(INFO, "\n      reset video_writer_");
   }
   if (!video_writer_->isOpened()) {
       video_writer_.reset(new VideoWriter("..//data//testgst-zhi-1920-30.avi",
                                           VideoWriter::fourcc('M', 'J', 'P', 'G'), 30,
                                          videoSize));
       LOGF(WARNING, "\n      error in reseting video_writer_ first time");
   }
   if (!video_writer_ && !video_writer_->isOpened()) {
       cerr << "!!!!!!error: cant open video writer\n" << flush;
       LOGF(WARNING, "\n!!!!!!error: cant open video writer");
       return 1;
   }
   if (video_cap_ && video_cap_->isOpened() && video_writer_ && video_writer_->isOpened()) {
       cout << "------camera is working\n" << flush;
       LOGF(INFO, "\n      both video_cap_ and video_writer_ are working");

       isCameraOpenedMutex_.lock();
       isCameraOpened_ = true;
       isCameraOpenedMutex_.unlock();

       isCameraStoppedMutex_.lock();
       isCameraStopped_ = false;
       isCameraStoppedMutex_.unlock();

       isCameraReadingStoppedMutex_.lock();
       isCameraReadingStopped_ = false;
       isCameraReadingStoppedMutex_.unlock();

       isCameraWritingStoppedMutex_.lock();
       isCameraWritingStopped_ = false;
       isCameraWritingStoppedMutex_.unlock();

       thread te(&CMyServer::camera_reading_and_enque, this);
       te.detach();
       LOGF(INFO, "\n      starts camera_reading_and_enque() thread");
       thread td(&CMyServer::camera_deque_and_writing, this);
       td.detach();
       LOGF(INFO, "\n      starts camera_deque_and_writing() thread");
   }

   return 0;
}

int CMyServer::close_camera() {
   if (video_cap_ && video_cap_->isOpened() && video_writer_ && video_writer_->isOpened()) {
       isCameraStoppedMutex_.lock();
       isCameraStopped_ = true;
       isCameraStoppedMutex_.unlock();
       LOGF(INFO, "\n      wants to close camera");

       bool read_stopped = false, write_stopped = false;
       while (1) {
           isCameraReadingStoppedMutex_.lock();
           if (isCameraReadingStopped_)
               read_stopped = true;
           isCameraReadingStoppedMutex_.unlock();

           isCameraWritingStoppedMutex_.lock();
           if (isCameraWritingStopped_)
               write_stopped = true;
           isCameraWritingStoppedMutex_.unlock();

           if (write_stopped && read_stopped) {
//            if (read_stopped) {
//                video_cap_->release();
//                video_writer_->release();
               isCameraOpenedMutex_.lock();
               isCameraOpened_ = false;
               isCameraOpenedMutex_.unlock();
               LOGF(INFO, "\n      both thread are closed, free video_cap_ and video_writer_ and queue");
               cout << "------camera is closed\n" << flush;
               break;
           }
       }
       std::queue<Mat>().swap(video_frame_queue_);
       video_cap_.reset();
       video_writer_.reset();
   }
   return 0;
}

void MyCamera::camera_reading_and_enque() {
   if (!(video_cap_ && video_cap_->isOpened() && video_writer_ && video_writer_->isOpened())) {
       LOGF(WARNING, "\n!!!!!! video_cap_ and video_writer_ are not all available in camera_reading_and_enque()");
       return;
   }
   bool isend = false;
   Mat frame;

   while (1) {
       video_cap_->read(frame);
       if (!frame.empty()) {

           VideoFrameQueueMutex_.lock();
           if (video_frame_queue_.size() < maxQueueSize_)
               video_frame_queue_.push(frame);
           else {
               video_frame_queue_.pop();
               video_frame_queue_.push(frame);
               LOGF(WARNING, "\n      video_frame_queue_.size() is > maxQueueSize_");
           }
           VideoFrameQueueMutex_.unlock();
       }

       isCameraStoppedMutex_.lock();
       if (isCameraStopped_)
           isend = true;
       isCameraStoppedMutex_.unlock();

       if (isend) {
           isCameraReadingStoppedMutex_.lock();
           isCameraReadingStopped_ = true;
           isCameraReadingStoppedMutex_.unlock();
           cout << "------end camera reading\n" << flush;
           LOGF(INFO, "\n      camera_reading_and_enque() thread is stopped");
           break;
       }
   }
}

void MyCamera::camera_deque_and_writing() {
   if (!(video_cap_ && video_cap_->isOpened() && video_writer_ && video_writer_->isOpened())) {
       LOGF(WARNING, "\n!!!!!! video_cap_ and video_writer_ are not all available in camera_deque_and_writing()");
       return;
   }
   bool isend = false;
   Mat frame;
   int fram_num = 0, pic_num = 0;

   while (1) {
       VideoFrameQueueMutex_.lock();
       if (video_frame_queue_.size() > 0) {
           frame = video_frame_queue_.front();
           video_frame_queue_.pop();
       } else {
           frame.resize(0);
           LOGF(WARNING, "\n      video_frame_queue_.size() is <= 0");
       }
       VideoFrameQueueMutex_.unlock();

       if (!frame.empty()) {
           video_writer_->write(frame);

           if (fram_num % 20 == 0) {
               stringstream str;
               str << "..//test_pic//camera//test//z" << pic_num << ".png";
//                str << "..//test_pic//camera//test//z" << pic_num << ".jpg";
               //str << "..//test_pic//camera//test//d" << pic_num << ".png";
//                str << "..//test_pic//camera//test//d" << pic_num << ".jpg";
               cout << str.str() << endl;
               pic_num++;
               imwrite(str.str(), frame);
               LOGF(INFO, "\n      write one frame to %s", str.str());
           }

           fram_num++;
       }

       isCameraStoppedMutex_.lock();
       if (isCameraStopped_)
           isend = true;
       isCameraStoppedMutex_.unlock();

       if (isend) {
           fram_num = 0;
           isCameraWritingStoppedMutex_.lock();
           isCameraWritingStopped_ = true;
           isCameraWritingStoppedMutex_.unlock();
           cout << "------end camera writing\n" << flush;
           LOGF(INFO, "\n------camera_deque_and_writing() thread is stopped");
           break;
       }
   }
}

problem description:

  • use open_camera() to start save the rtsp steam and jpg pictures.
  • use close_camera() to stop it.
  • use two threads to read the frames, decode and save to jpg.
  • use this gstreamer pipeline for using NVDEC chip on NX to decode video frames.
  • when repeat this process:
    call open_camera(), then call close_camera() after 3 secs,
  • My code is taking up more and more memory using top to show it.
  • use function release() of videocapture also has memory leaks.
  • it comes out that only when videocapture using gstreamer has memory leaks,and only when repeating process:“reset(pipeline),and then reset()” has memory leaks.
  • videocapture without gstreamer has no leaks.
  • valgrind shows all memory leaks are in the boost or gstreamer.

thanks for helping!!!
your help really matters!!!

1 Like

Hi,
Please apply this patch to rebuild/replace gst-v4l2:
Jetson/L4T/r32.6.x patches - eLinux.org
[gstreamer] Memory leak in UDP streaming

And check if you still observe memory leak.