Cannot proceed to randomize camera on Isaac Sim

Omniverse Isaac Sim
1

Hello,

I am trying to generate synthetic data to finetune a model of mine, for that I use Isaac Sim and try to randomize a few props across a stage.
I have tried everything I found in the documentations for months but I still cannot proceed to randomize the camera !!!
I joined my code below and a few steps of what I do in the simulation itself.
So I drag & drop my stage, under window I add a script editor and put my script below, I press Ctrl + Enter. Under Synthetic Data Recorder I change the camera path to World/Camera, I select the options I want in Parameters, I put the right output directory and then I press Start.
Please help me pinpoint what is wrong with my approach.

Thank you in advance for the time and help I may get !

Kaan Ucar

import asyncio
import itertools
import os
import numpy as np
import omni.replicator.core as rep
import omni.usd
from omni.isaac.core.utils.nucleus import get_assets_root_path
from pxr import Gf, Usd, UsdGeom, UsdLux

with rep.new_layer():

# Add Default Light
distance_light = rep.create.light(rotation=(315,0,0), intensity=100, light_type="distant")

# Setup camera and attach it to render product
stage = omni.usd.get_context().get_stage()
camera = stage.DefinePrim("/World/Camera", "Camera")
if not camera.GetAttribute("xformOp:translate"):
    UsdGeom.Xformable(camera).AddTranslateOp()
if not camera.GetAttribute("xformOp:orient"):
    UsdGeom.Xformable(camera).AddOrientOp()


writer = rep.WriterRegistry.get("BasicWriter")
writer.initialize(output_dir="C:/Users/naha/Desktop/sim_data", rgb=True, bounding_box_2d_tight=True)
rp_persp = rep.create.render_product("/OmniverseKit_Persp", (512, 512), name="PerspView")
rp_cam = rep.create.render_product(str(camera.GetPath()), (512, 512), name="SphereView")
writer.attach([rp_cam, rp_persp])


#Path to props folder
PROPS = 'omniverse://localhost/Library/'

list_props = [
    PROPS + 'Agave.usd', 
    PROPS + 'Japanese_Flame.usd',
    PROPS+'Jungle_Flame.usd'
]

list_rocks = [
    PROPS+'rock.usd'
]

#Create objects
sphere = rep.create.sphere(semantics=[('class', 'UO')], position=(0, 500, 0), scale=0.4)
cube = rep.create.cube(semantics=[('class', 'UO')],  position=(0, 300 , 0) , scale=0.4)
plane = rep.create.plane(scale=100, position=(0, -240, 0), visible=True)

#Create objects from list
for prop in list_props:
    plant = rep.create.from_usd(prop, semantics=[('class', 'UO')])
    with plant:
        rep.modify.pose(position=(0, 400, 0))
        rep.physics.rigid_body(
            velocity=rep.distribution.uniform((-0,0,-0),(0,0,0)),
            angular_velocity=rep.distribution.uniform((-0,0,-100),(0,0,0))
        )

#Create objects from list
for prop in list_rocks:
    rock = rep.create.from_usd(prop, semantics=[('class', 'Rock')])
    with rock:
        rep.modify.pose(position=(0, 400, 0))
        rep.physics.rigid_body(
            velocity=rep.distribution.uniform((-0,0,-0),(0,0,0)),
            angular_velocity=rep.distribution.uniform((-0,0,-100),(0,0,0))
        )

#Define physics
with plane:
    rep.physics.collider()

shapes = rep.get.prims(semantics=[('class', 'UO')])
with shapes:
    rep.physics.rigid_body(
            velocity=rep.distribution.uniform((-0,0,-0),(0,0,0)),
            angular_velocity=rep.distribution.uniform((-0,0,-100),(0,0,0))
    )

rocks_shapes = rep.get.prims(semantics=[('class', 'Rock')])
with rocks_shapes:
    rep.physics.rigid_body(
            velocity=rep.distribution.uniform((-0,0,-0),(0,0,0)),
            angular_velocity=rep.distribution.uniform((-0,0,-100),(0,0,0))
    )

#Randomize objects
def get_shapes():
    # Get objects via semantics
    shapes = rep.get.prims(semantics=[('class', 'UO')])
    
    with shapes:
        # Randomize color
        rep.randomizer.color(colors=rep.distribution.uniform((0, 0, 0), (1, 1, 1)))
        
        # Randomize pos, rot and size
        rep.modify.pose(
            position=rep.distribution.uniform((-400, 110, -400), (400, 110, 400)),
            rotation=rep.distribution.uniform((0, -180, 0), (0, 180, 0)),
            scale=rep.distribution.uniform(0.4, 0.8)
        )
        
    return shapes.node

#Randomize rocks
def get_rocks_shapes():
    shapes = rep.get.prims(semantics=[('class', 'Rock')])
    with shapes:
        rep.modify.pose(
            position=rep.distribution.uniform((-400, 110, -400), (400, 110, 400)),
            rotation=rep.distribution.uniform((0, -180, 0), (0, 180, 0)),
            scale=rep.distribution.uniform(0.4, 0.8)
        )
    return shapes.node

Define function to randomize camera pose

def randomize_camera():
    cam_pos = rep.distribution.uniform((-400, 200, -400), (400, 500, 400)).sample()
    camera.GetAttribute("xformOp:translate").Set(Gf.Vec3d(*cam_pos))

    eye = Gf.Vec3d(*cam_pos)
    target = (0, 0, 0)
    up_axis = Gf.Vec3d(0, 0, 1)
    look_at_quatd = Gf.Matrix4d().SetLookAt(eye, target, up_axis).GetInverse().ExtractRotation().GetQuat()
    camera.GetAttribute("xformOp:orient").Set(Gf.Quatf(look_at_quatd))

#Register functions in the replicator randomizer
rep.randomizer.register(get_shapes)
rep.randomizer.register(get_rocks_shapes)
rep.randomizer.register(randomize_camera)

# Setup randomization
with rep.trigger.on_frame(num_frames=15):
    rep.randomizer.get_shapes()
    rep.randomizer.get_rocks_shapes()
    rep.randomizer.randomize_camera()
3

Hi there,

to randomize the camera using the replicator modifyers you should create your camera with replicator api: rep.create.camera(). This will add a transform offset which will make sure the camera will look at the requested location when randomized.

Otherwise you can also use custom USD/Isaac Sim API randomizers and use replicator to capture the data:

1 Like
4

Thank you for your answer !

5

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.