Empty ndarray is returned from camera.get_rgb() on IsaacSim API

Omniverse Isaac Sim
1

Hi, I am going to build a stereo camera setup which rotates around the origin of the space and captures images on IsaacSim.

I successfully added two cameras via Python API, and made it sure that I can see it works correctly when I open Camera View on GUI. However, I cannot obtain image ndarray by using camera.get_rgb(). Is there anything I missed to run before using get_rgb() or any other reason it did not work?

Here is my code.

from omni.isaac.kit import SimulationApp
import numpy as np
import cv2
import yaml


# Launch simulation
config = {
        "window_width": "1000",
        "window_height": "700",
        "headless": False,
    }
simulation_app = SimulationApp(config)


import omni
from omni.isaac.core.utils.stage import add_reference_to_stage
from omni.isaac.core import SimulationContext, utils
from omni.isaac.core import World
from omni.isaac.core.physics_context import PhysicsContext
import omni.isaac.core.utils.prims as prim_utils
from omni.isaac.sensor import Camera
import omni.isaac.core.utils.numpy.rotations as rot_utils


class StereoCamera:
    def __init__(self):
        self.distance = 0.3
        self.rz = 30 # radius in z surface
        self.next_height = 30

        self.resolution = (256, 256)
        
        #self.z_rot = 0 #  (degree)angle in z rotation axis
        self.initial_pos = np.array([self.rz, 0, self.next_height])
        #self.initial_rot = rot_utils.euler_angles_to_quats(np.array([0, 45, 180]), degrees=True)


        self.camera_main = Camera(
            prim_path="/World/Camera/camera_main",
            position=self.initial_pos,
            frequency=20,
            resolution=self.resolution,
            orientation=self.calc_gazing_rot(self.initial_pos),
        )

        self.camera_right = Camera(
            prim_path="/World/Camera/camera_right",
            position=self.sub_camera_pos,
            frequency=20,
            resolution=self.resolution,
            orientation=self.rot,
        )

    def initialize_cameras(self):
        self.camera_main.initialize()
        self.camera_right.initialize()
        
        

    def calc_gazing_rot(self, pos, z_offset=2): # calc rotation gaizing the origin of the world(return quaternion)
        x, y, z = pos
        rz = np.arctan2(y, x)
        rx = np.pi
        ry = np.arctan2(z-z_offset, -np.sqrt(x**2 + y**2))
        return rot_utils.euler_angles_to_quats(np.array([rx, ry, rz]), degrees=False)

    @property
    def height(self):
        return self.pos[2]

    @property
    def z_rot(self): # degree
        x, y, z = self.pos
        return np.degrees(np.arctan2(y, x))

    @property
    def pos(self):
        return self.camera_main.get_world_pose()[0]

    @property  
    def rot(self): # quotation
        return  self.camera_main.get_world_pose()[1]
    
    @property
    def sub_camera_pos(self):
        rot_z = rot_utils.quats_to_euler_angles(self.rot, degrees=False)[2]
        return self.pos + self.distance* np.array([np.sin(rot_z), -np.cos(rot_z), 0])
    
    def update_z_rot(self, z_pitch_euler=0.1):
        new_z_rot = self.z_rot + z_pitch_euler
        x = self.rz * np.cos(np.radians(new_z_rot))
        y = self.rz * np.sin(np.radians(new_z_rot))
        z = self.next_height
        new_pos = np.array([x, y, z])
        new_rot = self.calc_gazing_rot(new_pos)

        self.camera_main.set_world_pose(new_pos, new_rot)
        self.camera_right.set_world_pose(self.sub_camera_pos, self.rot)

   

def main():
    my_world = World(stage_units_in_meters=1.0)
    simulation_context = SimulationContext(set_defaults=False)
    physics_context = my_world.get_physics_context()
    physics_context.enable_gpu_dynamics(True)

    stereo_camera = StereoCamera()
    print("Cemaras are created: ", stereo_camera.camera_main, stereo_camera.camera_right)

 
    # Reset and initialize
    my_world.reset()
    stereo_camera.initialize_cameras()
    my_world.step(render=True)    

    # Test Capture Camera Images
    image_main = stereo_camera.camera_main.get_current_frame()["rgba"]
    image_right = stereo_camera.camera_right.get_current_frame()["rgba"]
    print(stereo_camera.camera_main.get_rgb())
    
  
    while simulation_app.is_running():
        # Capture camera image


        # Update camera position
        stereo_camera.update_z_rot(1)
        my_world.step(render=True)
        ## Height Control and Terminate condition
        if abs(stereo_camera.z_rot) <= 0.01:
            if stereo_camera.height <= 0:
                break

            stereo_camera.next_height -= 5
            
            

    simulation_app.close()


if __name__ == '__main__':
    main()
2

Hi,
I had a similar issue using camera.get_rgb(). I solved it by waiting some frames before calling it.
Try with the following:

i = 0
while simulation_app.is_running():
        my_world.step(render=True)
        if world.is_playing() and i > 50:
                rgb = camera.get_rgb()
        i += 1

It’s most surely not the best solution, but it worked in my case. Hope it helps

3

Thank you for your reply, I ends up with the same answer!

4

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.