Articulation controller works in standalone python script but not in extension

Omniverse Isaac Sim
, , ,
1

Hello,

I am currently working on implementing a robotics application in Omniverse. Initially, I started the development by creating a standalone script. However, now I want to integrate this functionality into an extension. Unfortunately, I’m encountering an issue where the rmpflow controller behaves unexpectedly in the extension.

To provide a clearer understanding, I have recorded two videos. The first video demonstrates the application running as a standalone script:

The second video showcases the same code integrated into an extension:

For your convenience, here is the code for the standalone script:

simulation_app = SimulationApp({"headless": False})

# loading of scene
def setup_scene(world):
    root_dir = Path(__file__).parents[4]
    #scene_config_dir = r"C:\Users\user\Documents\KIRoPro_Git\Extensions\test_scene_config_round_gripper.json"
    scene_config_dir = r"C:\Users\user\Documents\KIRoPro_Git\Extensions\test_scene_config.json"
    scenePath = define_stage(scenePath="/physicsScene", tsps=240)
    world.scene.add_default_ground_plane()
    list_objects_scene = load_scene_from_json(world, root_dir, scene_config_dir)
    define_object_physics(list_objects_scene, root_dir, scene_config_dir, world)
    robot_dict = load_robot_from_json(world, root_dir, scene_config_dir)

    return robot_dict

my_world = World(stage_units_in_meters=1.0)
robot_dict = setup_scene(my_world)

my_robot = robot_dict['list_gripper'][0] 


# robot commands
command_list = RobotProgramm(standardt_endeffector_orientation=euler_angles_to_quat(np.array([0, np.pi/2, 0])))

#"""
command_list.addOpenGripper()
command_list.addMoveJ(np.array([0,0,0.9]))
command_list.addMoveJToObject("Aufnahmepunkt_1", np.array([0,0,0.05]))
command_list.addMoveJToObject("Aufnahmepunkt_1", np.array([0,0,0]))
command_list.addCloseGripper()
command_list.addWait(pause_in_seconds=0.5)
command_list.addMoveJToObject("Aufnahmepunkt_1", np.array([0,0,0.05]))
command_list.addMoveJToObject("Ablagepunkt_1_3", np.array([0,0,0.05]))
command_list.addMoveJToObject("Ablagepunkt_1_3", np.array([0,0,0]))
command_list.addOpenGripper()
command_list.addWait(pause_in_seconds=0.5)
command_list.addMoveJToObject("Ablagepunkt_1_3", np.array([0,0,0.05]))
command_list.addMoveJToObject("Aufnahmepunkt_2", np.array([0,0,0.05]))
command_list.addMoveJToObject("Aufnahmepunkt_2", np.array([0,0,0]))
command_list.addCloseGripper()
command_list.addWait(pause_in_seconds=0.5)
command_list.addMoveJToObject("Aufnahmepunkt_2", np.array([0,0,0.05]))
command_list.addMoveJToObject("Ablagepunkt_2_3", np.array([0,0,0.05]))
command_list.addMoveJToObject("Ablagepunkt_2_3", np.array([0,0,0]))
command_list.addOpenGripper()
command_list.addWait(pause_in_seconds=0.5)
command_list.addMoveJToObject("Ablagepunkt_2_3", np.array([0,0,0.05]))
command_list.addMoveJToObject("Aufnahmepunkt_3", np.array([0,0,0.05]))
command_list.addMoveJToObject("Aufnahmepunkt_3", np.array([0,0,0]))
command_list.addCloseGripper()
command_list.addWait(pause_in_seconds=0.5)
command_list.addMoveJToObject("Aufnahmepunkt_3", np.array([0,0,0.05]))
command_list.addMoveJToObject("Ablagepunkt_3_3", np.array([0,0,0.05]))
command_list.addMoveJToObject("Ablagepunkt_3_3", np.array([0,0,0]))
command_list.addOpenGripper()
command_list.addWait(pause_in_seconds=0.5)
command_list.addMoveJToObject("Ablagepunkt_3_3", np.array([0,0,0.05]))
command_list.addMoveJ(np.array([0,0,0.9]))
#"""

# initialize the controller
name = "test_controller"
cspace_controller=RMPFlowController(
                name=name + "_cspace_controller", robot_articulation=my_robot
            )
my_controller = KIRoProController(name=name, 
                                  gripper=my_robot.gripper, 
                                  cspace_controller=cspace_controller,
                                  command_list=command_list,
                                  world=my_world
                                  )

articulation_controller = my_robot.get_articulation_controller()

# sim step
i = 0
while simulation_app.is_running():
    my_world.step(render=True)
    if my_world.is_playing():
        if my_world.current_time_step_index == 0:
            my_world.reset()
            #my_controller.reset()

        # get joint state and calculate actions
        joints_state = my_robot.get_joints_state()
        actions = my_controller.forward(
            current_joint_positions=joints_state.positions,
            end_effector_offset=np.array([0, 0, 0.152]), #0.152
            end_effector_orientation=euler_angles_to_quat(np.array([0, np.pi/2, 0]))
        )

        articulation_controller.apply_action(actions)

simulation_app.close()

Additionally, here is the code for the main extension file. I based it on the bin filling Isaac example and incorporated my code into it:

class BinFilling(BaseSample):
    def __init__(self) -> None:
        super().__init__()
    
    # loading of scene
    def setup_scene(self):
        self.world = self.get_world()
        #root_dir = Path(__file__).parents[4]
        root_dir = r"C:\Users\user\Documents\KIRoPro_Git\Extensions"
        scene_config_dir = r"C:\Users\user\Documents\KIRoPro_Git\Extensions\test_scene_config_round_gripper.json"
        scenePath = define_stage(scenePath="/physicsScene", tsps=240)
        self.world.scene.add_default_ground_plane()
        list_objects_scene = load_scene_from_json(self.world, root_dir, scene_config_dir)
        define_object_physics(list_objects_scene, root_dir, scene_config_dir, self.world)
        self.robot_dict = load_robot_from_json(self.world, root_dir, scene_config_dir)

        return

    async def setup_post_load(self):
        self.my_robot = self.robot_dict['list_gripper'][0] 


        # robot commands
        command_list = RobotProgramm(standardt_endeffector_orientation=euler_angles_to_quat(np.array([0, np.pi/2, 0])))

        #"""
        command_list.addOpenGripper()
        command_list.addMoveJ(np.array([0,0,0.9]))
        command_list.addMoveJToObject("Aufnahmepunkt_1", np.array([0,0,0.05]))
        command_list.addMoveJToObject("Aufnahmepunkt_1", np.array([0,0,0]))
        command_list.addCloseGripper()
        command_list.addWait(pause_in_seconds=0.5)
        command_list.addMoveJToObject("Aufnahmepunkt_1", np.array([0,0,0.05]))
        command_list.addMoveJToObject("Ablagepunkt_1_3", np.array([0,0,0.05]))
        command_list.addMoveJToObject("Ablagepunkt_1_3", np.array([0,0,0]))
        command_list.addOpenGripper()
        command_list.addWait(pause_in_seconds=0.5)
        command_list.addMoveJToObject("Ablagepunkt_1_3", np.array([0,0,0.05]))
        command_list.addMoveJToObject("Aufnahmepunkt_2", np.array([0,0,0.05]))
        command_list.addMoveJToObject("Aufnahmepunkt_2", np.array([0,0,0]))
        command_list.addCloseGripper()
        command_list.addWait(pause_in_seconds=0.5)
        command_list.addMoveJToObject("Aufnahmepunkt_2", np.array([0,0,0.05]))
        command_list.addMoveJToObject("Ablagepunkt_2_3", np.array([0,0,0.05]))
        command_list.addMoveJToObject("Ablagepunkt_2_3", np.array([0,0,0]))
        command_list.addOpenGripper()
        command_list.addWait(pause_in_seconds=0.5)
        command_list.addMoveJToObject("Ablagepunkt_2_3", np.array([0,0,0.05]))
        command_list.addMoveJToObject("Aufnahmepunkt_3", np.array([0,0,0.05]))
        command_list.addMoveJToObject("Aufnahmepunkt_3", np.array([0,0,0]))
        command_list.addCloseGripper()
        command_list.addWait(pause_in_seconds=0.5)
        command_list.addMoveJToObject("Aufnahmepunkt_3", np.array([0,0,0.05]))
        command_list.addMoveJToObject("Ablagepunkt_3_3", np.array([0,0,0.05]))
        command_list.addMoveJToObject("Ablagepunkt_3_3", np.array([0,0,0]))
        command_list.addOpenGripper()
        command_list.addWait(pause_in_seconds=0.5)
        command_list.addMoveJToObject("Ablagepunkt_3_3", np.array([0,0,0.05]))
        command_list.addMoveJ(np.array([0,0,0.9]))
        #"""
        # initialize the controller
        name = "test_controller"
        cspace_controller=RMPFlowController(
                name=name + "_cspace_controller", robot_articulation=self.my_robot
            )
        
        self.my_controller = KIRoProController(name=name, 
                                gripper=self.my_robot.gripper, 
                                cspace_controller=cspace_controller,
                                command_list=command_list,
                                world=self.world
                                )
        
        self.articulation_controller = self.my_robot.get_articulation_controller()
        
        return

    # sim step
    def _on_fill_bin_physics_step(self, step_size):
        if self.world.current_time_step_index == 0:
            self.world.reset_async()

        # get joint state and calculate actions
        joints_state = self.my_robot.get_joints_state()
        actions = self.my_controller.forward(
            current_joint_positions=joints_state.positions,
            end_effector_offset=np.array([0, 0, 0.152]), #0.152
            end_effector_orientation=euler_angles_to_quat(np.array([0, np.pi/2, 0]))
        )

        self.articulation_controller.apply_action(actions)
        return

    async def on_fill_bin_event_async(self):
        world = self.get_world()
        world.add_physics_callback("sim_step", self._on_fill_bin_physics_step)
        await world.play_async()
        return

    async def setup_pre_reset(self):
        world = self.get_world()
        if world.physics_callback_exists("sim_step"):
            world.remove_physics_callback("sim_step")
        self.my_controller.reset()
        self._added_screws = False
        return

    def world_cleanup(self):
        return

The only difference between the two approaches (except for the fact that the second version is executed as an extension) lies in the creation of the world instance and the world reset function.

I’m unsure about the source of the problem. The controller is initialized in the exact same way and with the same configuration files. I would greatly appreciate any insights or suggestions regarding this issue.

Thank you for your assistance.

Kind regards,

Axel

3

I found the issue, the physics dt in the physics scene (240 tsps) was different than the physics dt of the motion policy (60), as set in:

´´´
import omni.isaac.motion_generation as mg
self.articulation_rmp = mg.ArticulationMotionPolicy(robot_articulation, self.rmpflow, physics_dt)
´´´
Both need to have the same value, else rmpflow behaves like that. Interestingly, this inconsistency does not occur in the standalone version, despite having the same physics dt discrepancy.

In my opinion it would be convenient to have the ability to change the physics dt without the need to reconfigure the motion policy. This would provide greater flexibility and convenience when adjusting the simulation settings.

4

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.