Hi,
I’m trying to access in python 12 bits images from a camera (IMX485) on the AGX ORIN.
Untill now, I was using jetson.utils to access this camera, but I can only get 8bit images:
import jetson.utils
def display_csi_camera():
# Create the camera instance
camera = jetson.utils.gstCamera(3840, 2160, "csi://0")
# Create the display instance
display = jetson.utils.glDisplay()
# Main loop to capture and display frames from the camera
while display.IsOpen():
# Capture a frame from the camera
img, width, height = camera.CaptureRGBA(zeroCopy=1)
# Render the frame
display.RenderOnce(img, width, height)
# Update the window title with the current frames per second (FPS)
display.SetTitle("CSI Camera | {:.1f} FPS".format(display.GetFPS()))
# Check for user exit (Esc key)
if display.IsClosed():
break
# Call the main function to display the camera feed
if __name__ == "__main__":
display_csi_camera()
I didn’t find a way to get 12bit images using this method.
I also tried another method based on gst commands:
import gi
gi.require_version('Gst', '1.0')
from gi.repository import Gst, GLib
import numpy as np
import torch
import torchvision.transforms as transforms
class GStreamerCapture:
def __init__(self):
self.pipeline = None
self.appsink = None
self.frames = []
self.sample_count = 0
self.max_samples = 10 # You can adjust this
def start_pipeline(self, pipeline_str):
self.pipeline = Gst.parse_launch(pipeline_str)
self.appsink = self.pipeline.get_by_name("sink")
self.appsink.set_property("emit-signals", True)
self.appsink.set_property("max-buffers", 1)
self.appsink.connect("new-sample", self.on_new_sample)
self.pipeline.set_state(Gst.State.PLAYING)
def on_new_sample(self, appsink):
sample = appsink.emit("pull-sample")
if sample:
self.sample_count += 1
buffer = sample.get_buffer()
caps = sample.get_caps()
width = caps.get_structure(0).get_value("width")
height = caps.get_structure(0).get_value("height")
frame = np.ndarray(
(height, width, 3),
buffer=buffer.extract_dup(0, buffer.get_size()),
dtype=np.uint8,
)
self.frames.append(frame)
if self.sample_count >= self.max_samples:
self.pipeline.set_state(Gst.State.NULL)
GLib.MainLoop().quit()
return Gst.FlowReturn.OK
def capture_frames(self):
loop = GLib.MainLoop()
loop.run()
return self.frames
def convert_frames_to_tensors(frames):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
tensor_frames = []
for frame in frames:
image = torch.tensor(frame, dtype=torch.uint8)
image = image.permute(2, 0, 1) # Channels-first
image = image.float().div(255) # Normalize to [0, 1]
tensor_frames.append(image)
tensor_frames = torch.stack(tensor_frames).to(device)
return tensor_frames
if __name__ == "__main__":
Gst.init(None)
pipeline_str = (
"nvarguscamerasrc ! "
"video/x-raw(memory:NVMM), format=NV12, width=3840, height=2160, framerate=60/1 ! "
"nvvidconv ! video/x-raw, format=BGRx ! "
"videoconvert ! video/x-raw, format=BGR ! "
"appsink name=sink emit-signals=True max-buffers=1"
)
gstreamer_capture = GStreamerCapture()
gstreamer_capture.start_pipeline(pipeline_str)
frames = gstreamer_capture.capture_frames()
tensor_frames = convert_frames_to_tensors(frames)
print(tensor_frames.shape) # Print the shape of the tensor frames
I am not yet even sure it is correct because I realized I wasn’t even able to display a 12 bit image using gstreamer only:
gst-launch-1.0 nvarguscamerasrc ! 'video/x-raw(memory:NVMM), width=3840, height=2160, framerate=60/1, format=NV12' ! nvvidconv ! 'video/x-raw(memory:NVMM), format=NV12' ! autovideosink
Setting pipeline to PAUSED ...
Pipeline is live and does not need PREROLL ...
Setting pipeline to PLAYING ...
New clock: GstSystemClock
GST_ARGUS: Creating output stream
CONSUMER: Waiting until producer is connected...
GST_ARGUS: Available Sensor modes :
GST_ARGUS: 3840 x 2160 FR = 50,000000 fps Duration = 20000000 ; Analog Gain range min 1,000000, max 31,622776; Exposure Range min 450000, max 400000000;
GST_ARGUS: 3840 x 2160 FR = 59,999999 fps Duration = 16666667 ; Analog Gain range min 1,000000, max 31,622776; Exposure Range min 450000, max 400000000;
GST_ARGUS: 1920 x 1080 FR = 90,000001 fps Duration = 11111111 ; Analog Gain range min 1,000000, max 31,622776; Exposure Range min 450000, max 400000000;
GST_ARGUS: 1920 x 1080 FR = 90,000001 fps Duration = 11111111 ; Analog Gain range min 1,000000, max 31,622776; Exposure Range min 450000, max 400000000;
GST_ARGUS: Running with following settings:
Camera index = 0
Camera mode = 1
Output Stream W = 3840 H = 2160
seconds to Run = 0
Frame Rate = 59,999999
GST_ARGUS: Setup Complete, Starting captures for 0 seconds
GST_ARGUS: Starting repeat capture requests.
CONSUMER: Producer has connected; continuing.
ERROR: from element /GstPipeline:pipeline0/GstNvArgusCameraSrc:nvarguscamerasrc0: Internal data stream error.
Additional debug info:
gstbasesrc.c(3072): gst_base_src_loop (): /GstPipeline:pipeline0/GstNvArgusCameraSrc:nvarguscamerasrc0:
streaming stopped, reason not-negotiated (-4)
Execution ended after 0:00:00.647558925
Setting pipeline to NULL ...
GST_ARGUS: Cleaning up
CONSUMER: Done Success
GST_ARGUS: Done Success
Freeing pipeline ...
Does anyone know how to get these 12 bits images in python (ideally directly on the GPU to use with pytorch), or at least what is the correct gstreamer command to get them?
Thank you very much for your help!