Replicator - how to get cameraViewTransform

[peter.gaston]

Using Replicator, I’d like to capture the 6-D pose of an object in the local frame of the camera. However, the output type to generate “camera_params” is missing. What to do?

Using the UI Isaac Sim and python to drive replication, “Synthetic Data Recorder” it is simple to check off “camera params”. And indeed, this all works fine to allow me to find the pose of an object. However, the power of replicator is difficult/complex to reproduce.

So is there a way to get camera params with replicator?

(Yes, I know I can’t change the camera, so I can probably guesstimate the transform. Is that the only work around?)

[ahaidu]

Hi there,

can you check if the Offline Pose Estimation tutorial contains the function you are looking for.

Best,
Andrei

[peter.gaston]

IMHO there are currently roughly three ways to generate synthetic data:

1. From the UI only - I think this is deprecated for a year now.
2. Replicator Composer. A yaml file controls the generation. This is what I’m after. This doesn’t seem to allow a ‘camera params’ object to be output
3. Full-on python. Either the simple 10 line example (2. Synthetic Data Recorder — Omniverse Robotics documentation) - which works great but is highly limited, or the full-on off-line blah blah.py - which is powerful but complicated.

btw, thanks for your response!
p

[mona.jalal]

the links are not working. Could you please tell how I can access the camera parameters (K) if I have an script like this?

# run using path to ov/pkg/code-2022-3.1 && ./omni.code.sh --no-window --/omni/replicator/script=path to this script
# /home/mona/.local/share/ov/pkg/code-2022.3.3/omni.code.sh --no-window --/omni/replicator/script=/home/mona/SDG/omniverse-sdg/examples/offline_generation/fruit_box_generator_Code_offline.py
import datetime
now = datetime.datetime.now()
import omni.replicator.core as rep
import inspect

with rep.new_layer():
    CRATE = 'omniverse://localhost/NVIDIA/Samples/Marbles/assets/standalone/SM_room_crate_3/SM_room_crate_3.usd'    
    SURFACE = 'omniverse://localhost/NVIDIA/Assets/Scenes/Templates/Basic/display_riser.usd'    
    ENVS = 'omniverse://localhost/NVIDIA/Assets/Scenes/Templates/Interior/ZetCG_ExhibitionHall.usd'
    FRUIT_PROPS = {
        'apple': 'omniverse://localhost/NVIDIA/Assets/ArchVis/Residential/Food/Fruit/Apple.usd',
        'avocado': 'omniverse://localhost/NVIDIA/Assets/ArchVis/Residential/Food/Fruit/Avocado01.usd',
        'kiwi': 'omniverse://localhost/NVIDIA/Assets/ArchVis/Residential/Food/Fruit/Kiwi01.usd',
        'lime': 'omniverse://localhost/NVIDIA/Assets/ArchVis/Residential/Food/Fruit/Lime01.usd',
        'lychee': 'omniverse://localhost/NVIDIA/Assets/ArchVis/Residential/Food/Fruit/Lychee01.usd',
        'pomegranate': 'omniverse://localhost/NVIDIA/Assets/ArchVis/Residential/Food/Fruit/Pomegranate01.usd',
        'onion': 'omniverse://localhost/NVIDIA/Assets/ArchVis/Residential/Food/Vegetables/RedOnion.usd',
        'lemon': 'omniverse://localhost/NVIDIA/Assets/ArchVis/Residential/Decor/Tchotchkes/Lemon_01.usd',
        'orange': 'omniverse://localhost/NVIDIA/Assets/ArchVis/Residential/Decor/Tchotchkes/Orange_01.usd'    }
    
    def random_props(file_name, class_name, max_number=1, one_in_n_chance=3):
        instances = rep.randomizer.instantiate(file_name, size=max_number, mode='scene_instance')
        print(file_name)
        with instances:
            rep.modify.semantics([('class', class_name)])
            rep.modify.pose(
                position=rep.distribution.uniform((-8, 5, -25), (8, 30, 25)),
                rotation=rep.distribution.uniform((-180,-180, -180), (180, 180, 180)),
                scale = rep.distribution.uniform((0.8), (1.2)), 
            )

            # rep.modify.visibility(rep.distribution.choice([True],[False]*(one_in_n_chance)))
            a = [True]
            a.extend([False]*(one_in_n_chance))
            rep.modify.visibility(rep.distribution.choice(a))
        return instances.node
    
    def sphere_lights(num):
        lights = rep.create.light(
            light_type="Sphere",
            temperature=rep.distribution.normal(6500, 500),
            intensity=rep.distribution.normal(30000, 5000),
            position=rep.distribution.uniform((-300, -300, -300), (300, 300, 300)),
            scale=rep.distribution.uniform(50, 100),
            count=num        )
        return lights.node    
    rep.randomizer.register(sphere_lights)
    
    env = rep.create.from_usd(ENVS)
    surface = rep.create.from_usd(SURFACE)
    with surface:
        rep.physics.collider()
    crate = rep.create.from_usd(CRATE)
    with crate:
        rep.physics.collider()
        rep.physics.mass(mass=10000)
        rep.modify.pose(
                position=(0, 20, 0),
                rotation=(0, 0, 90)
            )
        
    
    # camera = rep.randomizer.register.camera()
    camera = rep.create.camera()
    render_product = rep.create.render_product(camera, resolution=(1024, 1024))
    
    # anno_1 = rep.annotators.get("CameraParams").attach(render_product)
    # with open('/hdd/SDG_out/fruit_box/cam_K.txt', 'w') as file:
    #     file.write(anno_1)
    
    rep.randomizer.register(random_props)
    rep.randomizer.register(sphere_lights)
    
    # trigger on frame for an interval
    with rep.trigger.on_frame(num_frames=3):
        rep.modify.timeline(5, "frame")
        for n, f in FRUIT_PROPS.items():
            print("testing on frame")
            random_props(f, n)
        rep.randomizer.sphere_lights(5)
        with camera:
            rep.modify.pose(position=rep.distribution.uniform((-20, 90, -17), (10, 140, -15)), look_at=crate)
            # rep.modify.pose(position=rep.distribution.uniform((-20, 90, -17), (10, 140, -15)), look_at=(0, 20, 20))
            
    # initialize and attach writer

    writer = rep.WriterRegistry.get("BasicWriter")
    print('writer.initialize args: ', inspect.signature(writer.initialize))
    now = now.strftime("%Y-%m-%d_%H:%M:%S")
    output_dir = "/hdd/SDG_out/fruit_box/" + now
    writer.initialize(output_dir=output_dir, 
                      rgb=True,
                      bounding_box_2d_tight=True,
                      bounding_box_2d_loose=True,
                      semantic_segmentation=True,
                      instance_segmentation=True,
                      distance_to_camera=True,
                      distance_to_image_plane=True,
                      bounding_box_3d=True,
                      occlusion=True,
                      normals=True)
    writer.attach([render_product])
    


print('finished writing!!')
            
    
    

[mona.jalal]

with this code:

    camera = rep.create.camera()
    render_product = rep.create.render_product(camera, resolution=(1024, 1024))
    
    cam_params = rep.annotators.get("CameraParams").attach(render_product)
    with open('/hdd/SDG_out/fruit_box/cam_params.txt', 'w') as f:
        f.write(str(cam_params.get_data()))

everything is set to zero. So, not sure why proper values are not posted?

{'cameraAperture': array([0., 0.], dtype=float32), 'cameraApertureOffset': array([0., 0.], dtype=float32), 'cameraFisheyeLensP': array([], dtype=float32), 'cameraFisheyeLensS': array([], dtype=float32), 'cameraFisheyeMaxFOV': 0.0, 'cameraFisheyeNominalHeight': 0, 'cameraFisheyeNominalWidth': 0, 'cameraFisheyeOpticalCentre': array([0., 0.], dtype=float32), 'cameraFisheyePolynomial': array([], dtype=float32), 'cameraFocalLength': 0.0, 'cameraFocusDistance': 0.0, 'cameraFStop': 0.0, 'cameraModel': '', 'cameraNearFar': array([0., 0.], dtype=float32), 'cameraProjection': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]), 'cameraViewTransform': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]), 'metersPerSceneUnit': 0.0, 'renderProductResolution': array([0, 0], dtype=int32)}

My ideal goal is to be able to set the camera params to that of Intel RealSense D435.

[ahaidu]

Hi there, can you try calling rep.orchestrator.step() once before calling get_data()?

[leonardo.dinnouti]

Are there any other hints on how to get the correct camera parameters? I would like to know how to draw the 3D coordinates into the final 2D RGB images, but I don’t know how without the camera information.

I’m also receiving the data structure with all values set to zero:

{'cameraAperture': array([0., 0.], dtype=float32), 'cameraApertureOffset': array([0., 0.], dtype=float32), 'cameraFisheyeLensP': array([], dtype=float32), 'cameraFisheyeLensS': array([], dtype=float32), 'cameraFisheyeMaxFOV': 0.0, 'cameraFisheyeNominalHeight': 0, 'cameraFisheyeNominalWidth': 0, 'cameraFisheyeOpticalCentre': array([0., 0.], dtype=float32), 'cameraFisheyePolynomial': array([], dtype=float32), 'cameraFocalLength': 0.0, 'cameraFocusDistance': 0.0, 'cameraFStop': 0.0, 'cameraModel': '', 'cameraNearFar': array([0., 0.], dtype=float32), 'cameraProjection': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]), 'cameraViewTransform': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]), 'metersPerSceneUnit': 0.0, 'renderProductResolution': array([0, 0], dtype=int32)}

[ahaidu]

Hi there,

if you getting empty arrays it is most probably because orchestrator.step() has not been called to feed the annotators with new data.

Here is a script that can be called in the script editor and it should print you the camera parameters:

import asyncio
import omni.replicator.core as rep

cam1 = rep.create.camera(f_stop=1.8)
cam2 = rep.create.camera(position=(1, 0, 0), projection_type="fisheye_polynomial")

rp1 = rep.create.render_product(cam1, (512, 512))
rp2 = rep.create.render_product(cam2, (512, 512))

cam_params_annot1 = rep.annotators.get("CameraParams")
cam_params_annot1.attach(rp1)

cam_params_annot2 = rep.annotators.get("CameraParams")
cam_params_annot2.attach(rp2)

async def get_camera_params_async():
    # NOTE: step_async() is needed to feed the annotator with new data
    await rep.orchestrator.step_async()
    data1 = cam_params_annot1.get_data()
    print(f"data1={data1}")
    data2 = cam_params_annot2.get_data()
    print(f"data2={data2}")
    await asyncio.sleep(1)

task = asyncio.ensure_future(get_camera_params_async())

Output:

data1={'cameraAperture': array([20.955 , 15.2908], dtype=float32), 'cameraApertureOffset': array([0., 0.], dtype=float32), 'cameraFisheyeLensP': array([], dtype=float32), 'cameraFisheyeLensS': array([], dtype=float32), 'cameraFisheyeMaxFOV': 0.0, 'cameraFisheyeNominalHeight': 0, 'cameraFisheyeNominalWidth': 0, 'cameraFisheyeOpticalCentre': array([0., 0.], dtype=float32), 'cameraFisheyePolynomial': array([0., 0., 0., 0., 0.], dtype=float32), 'cameraFocalLength': 24.0, 'cameraFocusDistance': 400.0, 'cameraFStop': 1.7999999523162842, 'cameraModel': 'pinhole', 'cameraNearFar': array([1.e+00, 1.e+06], dtype=float32), 'cameraProjection': array([ 2.29062286e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
        0.00000000e+00,  2.29062286e+00,  0.00000000e+00,  0.00000000e+00,
        0.00000000e+00,  0.00000000e+00,  1.00000100e-06, -1.00000000e+00,
        0.00000000e+00,  0.00000000e+00,  1.00000100e+00,  0.00000000e+00]), 'cameraViewTransform': array([ 2.22044605e-16, -2.22044605e-16,  1.00000000e+00,  0.00000000e+00,
        1.00000000e+00,  4.93038066e-32, -2.22044605e-16,  0.00000000e+00,
        0.00000000e+00,  1.00000000e+00,  2.22044605e-16, -0.00000000e+00,
       -0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  1.00000000e+00]), 'metersPerSceneUnit': 1.0, 'renderProductResolution': array([512, 512], dtype=int32)}
data2={'cameraAperture': array([20.955 , 15.2908], dtype=float32), 'cameraApertureOffset': array([0., 0.], dtype=float32), 'cameraFisheyeLensP': array([-0.00037, -0.00074], dtype=float32), 'cameraFisheyeLensS': array([-0.00058, -0.00022,  0.00019, -0.0002 ], dtype=float32), 'cameraFisheyeMaxFOV': 200.0, 'cameraFisheyeNominalHeight': 1216, 'cameraFisheyeNominalWidth': 1936, 'cameraFisheyeOpticalCentre': array([970.94244, 600.3748 ], dtype=float32), 'cameraFisheyePolynomial': array([0.     , 0.00245, 0.     , 0.     , 0.     , 0.     ],
      dtype=float32), 'cameraFocalLength': 24.0, 'cameraFocusDistance': 400.0, 'cameraFStop': 0.0, 'cameraModel': 'fisheyePolynomial', 'cameraNearFar': array([1.e+00, 1.e+06], dtype=float32), 'cameraProjection': array([ 2.29062286e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
        0.00000000e+00,  2.29062286e+00,  0.00000000e+00,  0.00000000e+00,
        0.00000000e+00,  0.00000000e+00,  1.00000100e-06, -1.00000000e+00,
        0.00000000e+00,  0.00000000e+00,  1.00000100e+00,  0.00000000e+00]), 'cameraViewTransform': array([ 2.22044605e-16, -2.22044605e-16,  1.00000000e+00,  0.00000000e+00,
        1.00000000e+00,  4.93038066e-32, -2.22044605e-16,  0.00000000e+00,
        0.00000000e+00,  1.00000000e+00,  2.22044605e-16, -0.00000000e+00,
       -2.22044605e-16,  2.22044605e-16, -1.00000000e+00,  1.00000000e+00]), 'metersPerSceneUnit': 1.0, 'renderProductResolution': array([512, 512], dtype=int32)}

[zackliu2017]

hi ,Have you found a solution