Hello everyone,
I’m currently facing an issue with high latency when using a CSI camera with GStreamer on my Jetson Nano. The latency is approximately 2 seconds, which is significantly higher compared to using a USB camera where I experience no latency at all.
In my program, where I’m encountering this latency, I utilize a TensorRT model that runs on the GPU. Interestingly, when I run a simple program with just GStreamer and the CSI camera, there’s no noticeable latency. This leads me to believe that the issue might be related to the interaction between the TensorRT model and the GStreamer pipeline when using the CSI camera.
I’m working in an environment with JetPack 4.6.1. Has anyone else experienced similar issues or have insights on why this might be happening and how to resolve it? Any suggestions or advice would be greatly appreciated.
here is the whole code when there is latency :
import sys
import cv2
import imutils
from yoloDet import YoloTRT
def gstreamer_pipeline(
sensor_id=0,
capture_width=1920,
capture_height=1080,
display_width=960,
display_height=540,
framerate=24,
flip_method=1,
):
return (
"nvarguscamerasrc sensor-id=%d ! "
"video/x-raw(memory:NVMM), width=(int)%d, height=(int)%d, framerate=(fraction)%d/1 ! "
"nvvidconv flip-method=%d ! "
"video/x-raw, width=(int)%d, height=(int)%d, format=(string)BGRx ! "
"videoconvert ! "
"video/x-raw, format=(string)BGR ! appsink"
% (
sensor_id,
capture_width,
capture_height,
framerate,
flip_method,
display_width,
display_height,
)
)
# use path for library and engine file
model = YoloTRT(library="yolov5/build/libmyplugins.so", engine="yolov5/build/best_yolov5_300.engine", conf=0.5, yolo_ver="v5")
cap = cv2.VideoCapture(gstreamer_pipeline(flip_method=0), cv2.CAP_GSTREAMER)
while True:
ret, frame = cap.read()
frame = imutils.resize(frame, width=600)
detections, t = model.Inference(frame)
# for obj in detections:
# print(obj['class'], obj['conf'], obj['box'])
# print("FPS: {} sec".format(1/t))
fps = 1.0 / t
fps_text = "FPS: {:.2f}".format(fps)
cv2.putText(frame, fps_text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255,0),2)
cv2.imshow("Output", frame)
key = cv2.waitKey(1)
if key == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
code without latency (from github CSI Camera) :
# MIT License
# Copyright (c) 2019-2022 JetsonHacks
# Using a CSI camera (such as the Raspberry Pi Version 2) connected to a
# NVIDIA Jetson Nano Developer Kit using OpenCV
# Drivers for the camera and OpenCV are included in the base image
import cv2
"""
gstreamer_pipeline returns a GStreamer pipeline for capturing from the CSI camera
Flip the image by setting the flip_method (most common values: 0 and 2)
display_width and display_height determine the size of each camera pane in the window on the screen
Default 1920x1080 displayd in a 1/4 size window
"""
def gstreamer_pipeline(
sensor_id=0,
capture_width=1920,
capture_height=1080,
display_width=960,
display_height=540,
framerate=24,
flip_method=1,
):
return (
"nvarguscamerasrc sensor-id=%d ! "
"video/x-raw(memory:NVMM), width=(int)%d, height=(int)%d, framerate=(fraction)%d/1 ! "
"nvvidconv flip-method=%d ! "
"video/x-raw, width=(int)%d, height=(int)%d, format=(string)BGRx ! "
"videoconvert ! "
"video/x-raw, format=(string)BGR ! appsink"
% (
sensor_id,
capture_width,
capture_height,
framerate,
flip_method,
display_width,
display_height,
)
)
def show_camera():
window_title = "CSI Camera"
# To flip the image, modify the flip_method parameter (0 and 2 are the most common)
print(gstreamer_pipeline(flip_method=0))
video_capture = cv2.VideoCapture(gstreamer_pipeline(flip_method=0), cv2.CAP_GSTREAMER)
if video_capture.isOpened():
try:
window_handle = cv2.namedWindow(window_title, cv2.WINDOW_AUTOSIZE)
while True:
ret_val, frame = video_capture.read()
# Check to see if the user closed the window
# Under GTK+ (Jetson Default), WND_PROP_VISIBLE does not work correctly. Under Qt it does
# GTK - Substitute WND_PROP_AUTOSIZE to detect if window has been closed by user
if cv2.getWindowProperty(window_title, cv2.WND_PROP_AUTOSIZE) >= 0:
cv2.imshow(window_title, frame)
else:
break
keyCode = cv2.waitKey(10) & 0xFF
# Stop the program on the ESC key or 'q'
if keyCode == 27 or keyCode == ord('q'):
break
finally:
video_capture.release()
cv2.destroyAllWindows()
else:
print("Error: Unable to open camera")
if __name__ == "__main__":
show_camera()