How to get bounding box 2d of projection material

[feneu005]

Hi NVIDIA Community,

i am using replicator projection material to project an image on an asset.
Now, i want to get the bounding box of the asset and the bounding box of the image, which is projected on the asset, using the replicator annotator bounding_box_2d_loose.

As result i get the same bounding box points for the asset and the projected image on the asset, and not two individual bounding boxes.

How can i get the bounding box of the projection?

[ahaidu]

Hi there,

does it work if you semantically annotate the projected material?

• Replicator Materials — Omniverse Extensions

Best,
Andrei

[feneu005]

Hi Andrei,
here is an basic example of what i tried to get the bounding box for the projection material:

import os
# simulation app
from omni.isaac.kit import SimulationApp
config = {
     'width': "1280",
     'height': "720",
     'headless': True,
     'fast_shutdown': True
}
simulation_app = SimulationApp(config)

# isaac sim imports 
import omni.replicator.core as rep
from omni.replicator.core import AnnotatorRegistry, BackendDispatch, Writer

rep.settings.set_stage_meters_per_unit(1.0)

#create light 
dome = rep.create.light(
      light_type = "dome"
)

#create camera
camera = rep.create.camera(position=(2, 0, 0), rotation= (0,0,0))
render_product = rep.create.render_product(camera, (1024, 1024))

texture_path = os.path.join(rep.example.TEXTURES_DIR, "smiley_albedo.png")

# create projection material
projection_asset = rep.create.plane(
    position=(0, 0, 0), 
    rotation=(0, 90, 0), 
    semantics = [('class', 'projection_asset')], 
    name = "projection_asset", 
    ) # asset to project on

proxy_cube = rep.create.cube(
    position=(1, 0, 0), 
    rotation=(0, 0, 0), 
    scale=(0.2, 0.2, 0.2), 
    visible =  False,
    semantics = [('class', 'proxy_cube')],
    name = "proxy_cube",
    #parent = projection_asset
    )

with projection_asset:
    rep.create.projection_material(proxy_prim = proxy_cube, semantics = [('class', 'projection_material')])

projection = rep.get.prims(semantics = [('class', 'projection_material')])
with projection:
        rep.modify.projection_material(diffuse=texture_path)
# Move the projector prims, and update the decal shell projectors
# bbox annotator
bbox_2d_loose = rep.AnnotatorRegistry.get_annotator("bounding_box_2d_loose", init_params={"semanticTypes": ["class"]})
bbox_2d_loose.attach(render_product)

# generate image and bbox output
writer = rep.WriterRegistry.get("BasicWriter")
writer.initialize(output_dir=os.path.join(os.path.dirname(os.path.realpath(__file__)), "output"), rgb=True, bounding_box_2d_loose = True)
writer.attach([render_product])

# step and get bbox data from annotator
rep.orchestrator.step()
data = bbox_2d_loose.get_data()

print("bbox projection_asset:  ", data["data"][0])

# only if proxy cube is not visible
print("bbox projection_material: ", data["data"][1])

"""
Output:
bbox projection_asset:   (0, 217, 217, 806, 806, -1.)
bbox projection_material:  (1, 217, 217, 806, 806, 0.)
"""

quit()
simulation_app.close()

[ahaidu]

Hi there,

seems to work just fine for me, here is a script you can test in the script editor:

import asyncio
import os
import omni.replicator.core as rep
import omni.usd

omni.usd.get_context().new_stage()
dome = rep.create.light(light_type = "dome")
texture_path = os.path.join(rep.example.TEXTURES_DIR, "smiley_albedo.png")

projection_asset = rep.create.plane(
    position=(0, 0, 0), 
    rotation=(0, 90, 0), 
    semantics = [('class', 'projection_asset')], 
    name = "projection_asset", 
    ) # asset to project on

proxy_cube = rep.create.cube(
    position=(1, 0, 0), 
    rotation=(0, 0, 0),
    scale=(0.2, 0.2, 0.2), 
    visible =  False,
    semantics = [('class', 'proxy_cube')],
    name = "proxy_cube",
    #parent = projection_asset
    )

with projection_asset:
    rep.create.projection_material(proxy_prim = proxy_cube, semantics = [('class', 'projection_material')])

projection = rep.get.prims(semantics = [('class', 'projection_material')])
with projection:
        rep.modify.projection_material(diffuse=texture_path)


async def get_data_async():
    rp = rep.create.render_product("/OmniverseKit_Persp", (512, 512))
    annot = rep.AnnotatorRegistry.get_annotator("bounding_box_2d_tight")
    annot.attach(rp)
    await rep.orchestrator.step_async()
    data = annot.get_data()
    print(data)

asyncio.ensure_future(get_data_async())

Here is the ouput with the two bounding boxes:

{'data': array([(0, 238, 225, 274, 286, -1.), (1, 252, 252, 258, 260,  0.)],
      dtype=[('semanticId', '<u4'), ('x_min', '<i4'), ('y_min', '<i4'), ('x_max', '<i4'), ('y_max', '<i4'), ('occlusionRatio', '<f4')]), 'info': {'bboxIds': array([0, 1], dtype=uint32), 'idToLabels': {'0': {'class': 'projection_asset'}, '1': {'class': 'projection_asset,projection_material'}}, 'primPaths': ['/Replicator/projection_asset_Xform', '/Replicator/projection_asset_Xform/Projection/proxy_cube_Xform']}}

[feneu005]

Thanks for your help Andrei,

i could solve the problem by using bounding_box_2d_tight instead of boundong_box_2d_loose annotator