from omni.kit.async_engine import run_coroutine
from omni.isaac.franka import Franka
from omni.isaac.core import World
from omni.isaac.core.utils.semantics import add_update_semantics, get_semantics
from omni.isaac.core.objects import DynamicCuboid
from omni.isaac.franka.controllers import PickPlaceController
from omni.isaac.sensor import Camera
import numpy as np
import asyncio
def get_input():
global target_cube, waiting_for_input
user_input = input("Would you like a blue / red cube?").lower()
target_cube = None
while target_cube is None:
if "blue" in user_input:
target_cube = blue_cube
print(f"GOT THE {target_cube}")
elif "red" in user_input:
target_cube = red_cube
print(f"GOT THE {target_cube}")
elif "stop" in user_input:
world.reset()
controller.reset()
break
else:
print("Invalid input. Please enter 'blue' or 'red'.")
target_cube = None
waiting_for_input = False
return target_cube
async def setup_scene():
global world, franka, blue_cube, red_cube, target_cube, controller, waiting_for_input, camera
waiting_for_input = True
target_cube = None
# 1. Create world FIRST
world = World()
world.scene.add_default_ground_plane()
# 2. Initialize physics BEFORE robot
await world.initialize_simulation_context_async() # Critical for physics
# 3. Add robot and cube
existing_franka = world.scene.get_object("franka")
if not existing_franka:
franka = Franka(prim_path="/World/Fancy_Franka", name="franka")
world.scene.add(franka)
else:
print("Object 'franka' already exists in the scene.")
existing_blue_cube = world.scene.get_object("blue_cube")
existing_red_cube = world.scene.get_object("red_cube")
if not existing_blue_cube:
blue_cube = DynamicCuboid(
prim_path="/World/blue_cube",
name="blue_cube",
position=np.array([0.3, 0.3, 0.3]),
scale=np.array([0.0515, 0.0515, 0.0515]),
color=np.array([0, 0, 1.0]),
)
world.scene.add(blue_cube)
print("Blue cube added!")
else:
print("Blue cube already exists in the scene.")
if not existing_red_cube:
red_cube = DynamicCuboid(
prim_path="/World/red_cube",
name="red_cube",
position=np.array([0.3, 0.5, 0.3]),
scale=np.array([0.0515, 0.0515, 0.0515]),
color=np.array([1.0, 0, 0]),
)
world.scene.add(red_cube)
print("Red cube added!")
else:
print("Red cube already exists in the scene.")
camera = world.scene.get_object("camera")
if not camera:
camera = Camera(prim_path="/World/Camera",
name="camera",
position=np.array([0.0, 0.0, 8.0]),
orientation=np.array([0.70711, 0, 0.70711, 0])
)
world.scene.add(camera)
print("Camera added!")
else:
print("Camera already exists in the scene.")
# 5. Create controller LAST
controller = PickPlaceController(
name="pick_place_controller",
gripper=franka.gripper,
robot_articulation=franka
)
add_update_semantics(prim=blue_cube.prim,
semantic_label="blue_cube",
type_label="class")
add_update_semantics(prim=red_cube.prim,
semantic_label="red_cube",
type_label="class")
print(get_semantics(blue_cube.prim))
controller.pause()
def physics_step(step_size):
global target_cube, waiting_for_input, camera
camera.add_bounding_box_2d_tight_to_frame()
data = camera.get_current_frame()
print(f"This is the bounding box: {data}\n")
print(f"This is the clipping range: {camera.get_clipping_range()}\n")
camera.set_clipping_range(0.1,1000000)
print(f"This is the blue_cube's semantics: {get_semantics(blue_cube.prim)}\n")
print("=== Physics step ===")
goal_position = np.array([0.4, -0.4, 0.3])
if controller.is_paused() and waiting_for_input:
controller.reset()
world.pause()
target_cube = get_input()
controller.resume()
world.play()
cube_position, _ = target_cube.get_world_pose()
current_joint_positions = franka.get_joint_positions()
print(f"Joint positions: {current_joint_positions}\n")
actions = controller.forward(
picking_position=cube_position,
placing_position=goal_position,
current_joint_positions=current_joint_positions,
)
franka.apply_action(actions)
if controller.is_done():
controller.pause()
waiting_for_input = True
# joint_positions=np.array([0.0, -1.0, 0.0, -2.2, 0.0, 2.4, 0.8, 0.04, 0.04])
# franka.set_joint_positions(joint_positions)
print("Controller just reset!")
async def run_simulation():
await world.play_async()
world.initialize_physics()
camera.initialize()
franka.initialize()
world.add_physics_callback("sim_step", physics_step)
# Initialize gripper AFTER simulation starts
franka.gripper.set_joint_positions(franka.gripper.joint_opened_positions)
await asyncio.sleep(1) # Allow gripper to settle
async def main():
await setup_scene() # First setup scene/robot
print("set up scene")
await run_simulation() # Then run simulation
await asyncio.sleep(10) # Keep simulation running
run_coroutine(main())
I am using Isaac Sim 4.2.0, and do not want to change to Isaac Sim 4.5.0 as it does not work for me. I am unable to get the camera to detect the objects it sees, even after updating semantics of the prims in my world. I am trying to use add_bounding_box_2d_tight_to_frame(), but it is still unable to output the details I need. The annotator currently outputs None. Help is really appreciated!
This is the documentation I have been following:
Isaac Sensor Extension [omni.isaac.sensor] — isaac_sim 4.2.0-rc.17 documentation
