Isaac Sim Version
5.0
Operating System
Ubuntu 22.04
Topic Description
Detailed Description
When generating camera data using Isaac sim with the opencv-fisheye lens distortion model, the images generated are not consistent with where the image points should be using cv2.projectPoints for the same extrinsic and intrinsic camera parameters.
I have a standalone script based on this tutorial to reproduce the issue, runnable standalone with python.sh from the Isaac Sim build folder:
from isaacsim import SimulationApp
simulation_app = SimulationApp(launch_config={"headless": False})
# example code starts here (from https://docs.isaacsim.omniverse.nvidia.com/latest/sensors/isaacsim_sensors_camera.html#opencv-fisheye)
import isaacsim.core.utils.numpy.rotations as rot_utils
import numpy as np
from isaacsim.core.api import World
from isaacsim.core.api.objects import DynamicCuboid
from isaacsim.core.utils.stage import add_reference_to_stage
from isaacsim.sensors.camera import Camera
from isaacsim.storage.native import get_assets_root_path
from PIL import Image, ImageDraw
# Desired image resolution, camera intrinsics matrix, and distortion coefficients
width, height = 1920, 1200
camera_matrix = [[455.8, 0.0, 943.8], [0.0, 454.7, 602.3], [0.0, 0.0, 1.0]]
distortion_coefficients = [0.05, 0.01, -0.003, -0.0005]
# Camera sensor size and optical path parameters. These parameters are not the part of the
# OpenCV camera model, but they are nessesary to simulate the depth of field effect.
#
# Note: To disable the depth of field effect, set the f_stop to 0.0. This is useful for debugging.
# Set pixel size (microns)
pixel_size = 3
# Set f-number, the ratio of the lens focal length to the diameter of the entrance pupil (unitless)
f_stop = 1.8
# Set focus distance (meters) - chosen as distance from camera to cube
focus_distance = 1.5
# Add a ground plane to the scene
usd_path = get_assets_root_path() + "/Isaac/Environments/Grid/default_environment.usd"
add_reference_to_stage(usd_path=usd_path, prim_path="/ground_plane")
# Add some cubes and a Camera to the scene
cube_1 = DynamicCuboid(
prim_path="/new_cube_1",
name="cube_1",
position=np.array([0, 0, 0.5]),
scale=np.array([1.0, 1.0, 1.0]),
size=1.0,
color=np.array([255, 0, 0]),
)
cube_2 = DynamicCuboid(
prim_path="/new_cube_2",
name="cube_2",
position=np.array([2, 0, 0.5]),
scale=np.array([1.0, 1.0, 1.0]),
size=1.0,
color=np.array([0, 255, 0]),
)
cube_3 = DynamicCuboid(
prim_path="/new_cube_3",
name="cube_3",
position=np.array([0, 4, 1]),
scale=np.array([2.0, 2.0, 2.0]),
size=1.0,
color=np.array([0, 0, 255]),
)
camera = Camera(
prim_path="/World/camera",
position=np.array([0.0, 0.0, 2.0]), # 1 meter away from the side of the cube
frequency=30,
resolution=(width, height),
orientation=rot_utils.euler_angles_to_quats(np.array([0, 90, 0]), degrees=True),
)
camera.initialize()
# Calculate the focal length and aperture size from the camera matrix
((fx, _, cx), (_, fy, cy), (_, _, _)) = camera_matrix # fx, fy are in pixels, cx, cy are in pixels
horizontal_aperture = pixel_size * width * 1e-6 # convert to meters
vertical_aperture = pixel_size * height * 1e-6 # convert to meters
focal_length_x = pixel_size * fx * 1e-6 # convert to meters
focal_length_y = pixel_size * fy * 1e-6 # convert to meters
focal_length = (focal_length_x + focal_length_y) / 2 # convert to meters
# Set the camera parameters, note the unit conversion between Isaac Sim sensor and Kit
camera.set_focal_length(focal_length)
camera.set_focus_distance(focus_distance)
camera.set_lens_aperture(f_stop)
camera.set_horizontal_aperture(horizontal_aperture)
camera.set_vertical_aperture(vertical_aperture)
camera.set_clipping_range(0.05, 1.0e5)
# Set the distortion coefficients
camera.set_opencv_fisheye_properties(cx=cx, cy=cy, fx=fx, fy=fy, fisheye=distortion_coefficients)
# example code ends here
import os
import omni.replicator.core as rep # https://docs.isaacsim.omniverse.nvidia.com/5.0.0/replicator_tutorials/tutorial_replicator_overview.html
rp = rep.create.render_product("/World/camera", (1920,1200))
writer = rep.writers.get("BasicWriter") # https://docs.omniverse.nvidia.com/py/replicator/1.11.35/source/extensions/omni.replicator.core/docs/API.html#basicwriter
out_dir = os.path.join(os.getcwd(), "replicator_output")
print(f"Output directory: {out_dir}")
writer.initialize(output_dir=out_dir, rgb=True)
writer.attach(rp)
rep.orchestrator.step(rt_subframes=16)
writer.detach()
rp.destroy()
rep.orchestrator.wait_until_complete()
import cv2
from scipy.spatial.transform import Rotation
img = cv2.imread(os.path.join(out_dir, "rgb_0000.png"))
cube_corners = np.array([
[0.5, 0.5, 0.5],
[-0.5, 0.5, 0.5],
[0.5, -0.5, 0.5],
[-0.5, -0.5, 0.5],
[0.5, 0.5, -0.5],
[-0.5, 0.5, -0.5],
[0.5, -0.5, -0.5],
[-0.5, -0.5, -0.5],
], dtype=np.float32)
cube1_corners = cube_corners + np.array([0, 0, 0.5])
cube2_corners = cube_corners + np.array([2, 0, 0.5])
cube3_corners = 2*cube_corners + np.array([0, 4, 1])
all_cube_corners = np.vstack((cube1_corners, cube2_corners, cube3_corners))
distortion_coefficients_cv = np.array(distortion_coefficients, dtype=np.float32)
isaac_to_cv2_mat = np.array([
[0, -1, 0],
[0, 0, -1],
[1, 0, 0]
], dtype=np.float32)
isaac_to_cv2 = Rotation.from_matrix(isaac_to_cv2_mat)
rot_mat = np.array([
[1, 0, 0],
[0, 0, 1],
[0, -1, 0]
], dtype=np.float32)
# cv2_r = Rotation.from_matrix(isaac_to_cv2_mat) * Rotation.from_euler('y', [90], degrees=True)
cv2_r = Rotation.from_matrix(rot_mat)
cam_rotation = Rotation.from_euler('y', [90], degrees=True)
# cam_rotation = Rotation.identity()
# cam_rotation = Rotation.from_euler('y', [30], degrees=True)
cam_position = np.array([0, 0, 2], dtype=np.float32)
rvec_rot = isaac_to_cv2 * cam_rotation.inv() * isaac_to_cv2.inv()
tvec = -rvec_rot.apply(cam_position @ isaac_to_cv2_mat.T)
image_points, _ = cv2.projectPoints(
all_cube_corners @ isaac_to_cv2_mat.T,
rvec_rot.as_rotvec(),
tvec,
np.array(camera_matrix),
distortion_coefficients_cv
)
for i, point in enumerate(image_points):
if i < 8:
color = (0,0,255)
elif i < 16:
color = (0,255,0)
else:
color = (255,0,0)
if np.any(point[0].astype(int)<0):
continue
cv2.circle(img, tuple(point[0].astype(int)), 5, (120,120,120), -1)
cv2.circle(img, tuple(point[0].astype(int)), 3, color, -1)
cv2.imwrite(os.path.join(out_dir, "projected_points.png"), img)
simulation_app.close()
If Isaac’s rendering was matching opencv, the dots on the image would line up with the corners of each cube:
fisheye:
pinhole:
I’m working on a use case where it’s important that the camera intrinsics model matches exactly with opencv, so the discrepancy here makes Isaac Sim unusable. Are the opencv-based rendering models supposed to match up exactly? Am I doing anything wrong in my snippet to reproduce this issue?
