Isaac 2022.1.1
I am doing reinforcement learning using images using Isaac sim. There is a problem that the camera image of the agent, which is the observed value, becomes the image of the perspective camera.
Below is the camera class.
class Camera:
def __init__(self, camera_prim_path: str, width: int, height: int, fov, near, far, headless: bool = False, path: str = "/World"):
"""
Args:
camera_prim_path: The camera prim path
width: The horizontal image resolution in pixels
height: The vertical image resolution in pixels
fov: The field of view of the camera
near: The near plane distance
far: The far plane distance
"""
from omni.isaac.core.utils.stage import add_reference_to_stage
from omni.isaac.core.utils.viewports import set_camera_view
from omni.isaac.synthetic_utils import SyntheticDataHelper
self.sd_helper = SyntheticDataHelper()
import omni.kit.viewport.utility
from omni.kit.viewport.utility import get_active_viewport_window
import omni.kit.commands
#from omni.kit.viewport.utility import get_active_viewport
from pxr import Sdf, Usd, UsdGeom
self._width = width
self._height = height
self.__fov = fov
self.__near = near
self.__far = far
# TODO: see why it is not 62 degrees the horizontalAperture !!!!
self.__aspect = self._width / self._height
self._view_matrix = None
self.camera_prim_path = camera_prim_path
fov_horizontal = self.__aspect * fov
focal_length = 1.88
attributes = {"horizontalAperture": 2*focal_length*math.tan(fov_horizontal*math.pi/180/2),
"verticalAperture": 2*focal_length*math.tan(fov*math.pi/180/2),
"focalLength": focal_length,
"clippingRange": (self.__near, self.__far)
}
self.stage = omni.usd.get_context().get_stage()
self.camera_prim = self.stage.GetPrimAtPath(self.camera_prim_path)
# Set as current camera
if headless:
"""
viewport_interface = omni.kit.viewport_legacy.get_viewport_interface()
self.viewport = viewport_interface.get_viewport_window()
"""
viewport_interface = omni.kit.viewport.utility.get_active_viewport()
self.viewport = omni.kit.viewport.utility.get_active_viewport()
#self.viewport = omni.kit.viewport.utilityget_active_viewport_window
#"""
else:
viewport_handle = omni.kit.viewport_legacy.get_viewport_interface().create_instance()
list_viewports = omni.kit.viewport_legacy.get_viewport_interface().get_instance_list()
new_viewport_name = omni.kit.viewport_legacy.get_viewport_interface().get_viewport_window_name(
viewport_handle
)
self.viewport = omni.kit.viewport_legacy.get_viewport_interface(
).get_viewport_window(viewport_handle)
window_width = 200
window_height = 200
self.viewport.set_window_size(window_width, window_height)
self.viewport.set_window_pos(
800, window_height*(len(list_viewports)-2))
#"""
self.viewport.set_active_camera(camera_prim_path)
#self.viewport.camera_path = camera_prim_path
self.viewport.set_texture_resolution((self._width,self._height))
active_viewport = omni.kit.viewport.utility.get_active_viewport()
self.sd_helper.initialize(["rgb"], self.viewport)
def get_image(self):
# Get ground truths
#active_viewport = omni.kit.viewport.utility.get_active_viewport()
#"""
gt = self.sd_helper.get_groundtruth(
[
"rgb",
# "depthLinear",
# "depth",
# "boundingBox2DTight",
# "boundingBox2DLoose",
# "instanceSegmentation",
# "semanticSegmentation",
# "boundingBox3D",
# "camera",
# "pose"
],
self.viewport,
verify_sensor_init=False,
wait_for_sensor_data=0.0
)
rgb = gt["rgb"]
return rgb
Below is an instance.
l_camera_prim_path = "/Fork/Fork_mini_1_twin_camera/Fork_mini_1/Frame/Camera"
#l_camera_prim_path = "/Fork/Fork_mini_1/Frame/Camera"
r_camera_prim_path = "/Fork/Fork_mini_1_twin_camera/Fork_mini_1/Frame/r_Camera"
#r_camera_prim_path = "/Fork/Fork_mini_1/Frame/r_Camera"
self._l_cameras = Camera(camera_prim_path=l_camera_prim_path,
width=256, height=256, fov=45, near=0.10, far=4, headless=True)
self._r_cameras = Camera(camera_prim_path=r_camera_prim_path,
width=256, height=256, fov=45, near=0.10, far=4, headless=True)
I checked the image like this as a test.
def get_observations(self):
# image
cams_l = self._l_cameras
img_l = Image.fromarray(cams_l.get_image()).convert("RGB")
# debug
# image save pass
img_path= f"/home/sfai20/python_ws/RL_simple_fork_Q2/img/image_{self.counter}.png"
img_l.save(img_path)
The result was a perspective camera image.
The agent has a camera like this.
How can I get the agent’s camera image?