Minimal you say……….
ok look. i know this is dirty code. i know it, so please dont judge me, but its at least easy to follow.
should be able to copy paste this in.
from isaacsim import SimulationApp
# --------------------------
# START SIMULATION
# --------------------------
# Non-headless mode so we can see the stage
simulation_app = SimulationApp({"headless": False, "enable_cameras": True})
from pxr import UsdGeom, Gf, Sdf, UsdShade
import numpy as np
import os
import random
import math
import omni.replicator.core as rep
from isaacsim.core.api.physics_context import PhysicsContext
from pxr import UsdGeom, Gf
from pxr import Sdf
import math
from omni.isaac.core import World
from pxr.PhysxSchema import PhysxSceneAPI
from isaacsim.core.utils.prims import create_prim
import isaacsim.core.utils.prims as prims_utils
bg_x = 1.2
bg_y = 0.9
percent_variance = 0.5
def constrain_multiple_rotations_callback(step_size):
"""
Leaving this in just in case its causing something im not aware of.
Looks at every object in the sim and applies hard rotation limits to certain axes.
"""
def general_setup():
world = World(
stage_units_in_meters=1.0,
physics_dt=1.0/120.0,
rendering_dt=1.0/30.0
)
PhysicsContext()
stage = world.scene.stage
# Define the physics scene prim (or get if it already exists)
physics_scene_path = "/physicsScene"#"/World/physicsScene"
physics_scene_prim = stage.GetPrimAtPath(physics_scene_path)
PhysxSceneAPI.Apply(physics_scene_prim)
# Explicitly create the solver attributes with the correct type (int)
physics_scene_prim.CreateAttribute("physx:numPositionIterations", Sdf.ValueTypeNames.Int).Set(16)
physics_scene_prim.CreateAttribute("physx:numVelocityIterations", Sdf.ValueTypeNames.Int).Set(4)
world = World(stage_units_in_meters=1.0)
world.reset()
return world, stage
# --------------------------
# GRAY CARD
# --------------------------
def create_gray_card():
"""
Creates a rectangular mesh representing an 18% grey card with a standard USD Preview Surface material.
Args:
stage: The USD stage to add the gray card to.
"""
gray_card_path = "/World/GrayCardMesh"
# Create the mesh primitive using the core utils
gray_card_prim = create_prim(
gray_card_path, "Mesh",
position=Gf.Vec3d(0, 0, 0),
scale=Gf.Vec3f(1.0, 1.0, 1.0)
)
# Define the mesh geometry
mesh_geom = UsdGeom.Mesh(gray_card_prim)
mesh_geom.GetPointsAttr().Set(
[
Gf.Vec3f(-bg_x/2, -bg_y/2, 0.0),
Gf.Vec3f(bg_x/2, -bg_y/2, 0.0),
Gf.Vec3f(bg_x/2, bg_y/2, 0.0),
Gf.Vec3f(-bg_x/2, bg_y/2, 0.0),
]
)
mesh_geom.GetFaceVertexCountsAttr().Set([4])
mesh_geom.GetFaceVertexIndicesAttr().Set([0, 1, 2, 3])
# Create and configure a USD Preview Surface material
material_path = Sdf.Path("/World/Looks/GrayCardMaterial")
material_prim = UsdShade.Material.Define(stage, material_path)
shader_path = material_path.AppendPath("Shader")
shader_prim = UsdShade.Shader.Define(stage, shader_path)
# Use the USD Preview Surface shader ID
shader_prim.CreateIdAttr("UsdPreviewSurface")
# Set the diffuse color to 18% grey
shader_prim.CreateInput("diffuseColor", Sdf.ValueTypeNames.Color3f).Set(Gf.Vec3f(0.18, 0.18, 0.18))
# Set the roughness to 1.0 for a fully matte surface
shader_prim.CreateInput("roughness", Sdf.ValueTypeNames.Float).Set(1.0)
# Set metallic to 0.0 for a non-metallic surface
shader_prim.CreateInput("metallic", Sdf.ValueTypeNames.Float).Set(0.0)
shader_prim.CreateInput("specular", Sdf.ValueTypeNames.Float).Set(1.0)
shader_prim.CreateInput("ior", Sdf.ValueTypeNames.Float).Set(1.0)
# Connect the shader to the material's surface output
material_prim.CreateSurfaceOutput().ConnectToSource(shader_prim.ConnectableAPI(), "surface")
# Bind the material to the mesh
UsdShade.MaterialBindingAPI(gray_card_prim).Bind(material_prim, UsdShade.Tokens.strongerThanDescendants)
xformable = UsdGeom.Xformable(gray_card_prim)
xformable.ClearXformOpOrder()
return gray_card_prim
def move_prim(path, x, y, z):
prim = stage.GetPrimAtPath(path)
prim.GetAttribute("xformOp:translate").Set((x, y, z))
# --------------------------
# ADD CAMERA
# --------------------------
camera_path = "/World/TopCamera"
def get_camera_params():
aperture_w = 6.4 # mm, default horizontal aperture for 4:3
aperture_h = 4.8 # mm, default vertical aperture for 4:3
focal_len = 12.0
# Compute FOV from lens
h_fov = 2 * math.atan((aperture_w / 2) / focal_len)
v_fov = 2 * math.atan((aperture_h / 2) / focal_len)
# Required height to see entire plane
required_height_w = (bg_x / 2.0) / math.tan(h_fov / 2.0)
required_height_h = (bg_y / 2.0) / math.tan(v_fov / 2.0)
max_height = max(required_height_w, required_height_h)
return aperture_w, aperture_h, focal_len, max_height
def setup_camera(aperture_w, aperture_h, focal_len, max_height, camera_path):
# Create or fetch camera
if not stage.GetPrimAtPath(camera_path):
cam = UsdGeom.Camera.Define(stage, camera_path)
else:
cam = UsdGeom.Camera(stage.GetPrimAtPath(camera_path))
# Set camera parameters
# Aperture
cam.GetHorizontalApertureAttr().Set(aperture_w)
cam.GetVerticalApertureAttr().Set(aperture_h)
# Focal Length
cam.GetFocalLengthAttr().Set(focal_len)
cam.GetClippingRangeAttr().Set(Gf.Vec2f(0.001, 1000000.0))
# Set transform: camera above plane along +Z, looking downward (-Z)
xform = UsdGeom.Xformable(cam)
xform.AddTranslateOp().Set(Gf.Vec3d(0, 0, max_height))
xform.AddRotateXOp().Set(0.0) # no rotation needed; default looks down -Z in Z-up
def set_camera_height(camera_height, camera_path):
cam = stage.GetPrimAtPath(camera_path)
xform = UsdGeom.Xformable(cam)
xform.GetTranslateOp().Set(Gf.Vec3d(0, 0, camera_height))
# --------------------------
# ADD LIGHTS
# --------------------------
RECT_LIGHT_PATH = "/World/RectLight"
SPHERE_LIGHT_PATH = "/World/SphereLight"
CYLINDER_X1_LIGHT_PATH = "/World/CylinderLightX1"
CYLINDER_X2_LIGHT_PATH = "/World/CylinderLightX2"
CYLINDER_Y1_LIGHT_PATH = "/World/CylinderLightY1"
CYLINDER_Y2_LIGHT_PATH = "/World/CylinderLightY2"
def create_rect_light(max_height):
"""Creates a RectLight and returns the prim object."""
rect_light_prim = prims_utils.create_prim(
prim_path=RECT_LIGHT_PATH,
prim_type="RectLight",
position=np.array([0.0, 0.0, max_height]),
)
return rect_light_prim
def create_sphere_light(max_height, radius=0.05):
"""Creates a SphereLight and returns the prim object."""
sphere_light_prim = prims_utils.create_prim(
prim_path=SPHERE_LIGHT_PATH,
prim_type="SphereLight",
position=np.array([0.0, 0.0, max_height]),
attributes={"inputs:radius": radius},
)
return sphere_light_prim
def create_cylinder_x1_light(max_height, length=1.0, radius=0.05):
"""Creates a CylinderLight and returns the prim object."""
cylinder_x1_light_prim = prims_utils.create_prim(
prim_path=CYLINDER_X1_LIGHT_PATH,
prim_type="CylinderLight",
position=np.array([0.0, 0.0, max_height]),
attributes={
"inputs:length": length,
"inputs:radius": radius,
},
)
return cylinder_x1_light_prim
def create_cylinder_x2_light(max_height, length=1.0, radius=0.05):
"""Creates a CylinderLight and returns the prim object."""
cylinder_x2_light_prim = prims_utils.create_prim(
prim_path=CYLINDER_X2_LIGHT_PATH,
prim_type="CylinderLight",
position=np.array([0.0, 0.0, max_height]),
attributes={
"inputs:length": length,
"inputs:radius": radius,
},
)
return cylinder_x2_light_prim
def create_cylinder_y1_light(max_height, length=1.0, radius=0.05):
"""Creates a CylinderLight and returns the prim object."""
cylinder_y1_light_prim = prims_utils.create_prim(
prim_path=CYLINDER_Y1_LIGHT_PATH,
prim_type="CylinderLight",
position=np.array([0.0, 0.0, max_height]),
attributes={
"inputs:length": length,
"inputs:radius": radius,
},
)
# Apply a 90-degree rotation around the Y-axis to make it vertical
xformable = UsdGeom.Xformable(cylinder_y1_light_prim)
xformable.AddRotateZOp().Set(90.0)
return cylinder_y1_light_prim
def create_cylinder_y2_light(max_height, length=1.0, radius=0.05):
"""Creates a CylinderLight and returns the prim object."""
cylinder_y2_light_prim = prims_utils.create_prim(
prim_path=CYLINDER_Y2_LIGHT_PATH,
prim_type="CylinderLight",
position=np.array([0.0, 0.0, max_height]),
attributes={
"inputs:length": length,
"inputs:radius": radius,
},
)
# Apply a 90-degree rotation around the Y-axis to make it vertical
xformable = UsdGeom.Xformable(cylinder_y2_light_prim)
xformable.AddRotateZOp().Set(90.0)
return cylinder_y2_light_prim
def set_light_params(prim, new_position, new_attributes):
# Set the new position
xformable = UsdGeom.Xformable(prim)
translate_op = xformable.GetTranslateOp()
translate_op.Set(Gf.Vec3d(*new_position))
# Set the new attributes
# The attributes dictionary can contain `inputs:radius`, `inputs:intensity`, etc.
for attr_name, attr_value in new_attributes.items():
attr = prim.GetAttribute(attr_name)
if attr.IsValid():
attr.Set(attr_value)
else:
print(f"Warning: Attribute '{attr_name}' not found")
def set_light_attributes(prim, new_attributes):
# Set the new attributes
# The attributes dictionary can contain `inputs:radius`, `inputs:intensity`, etc.
for attr_name, attr_value in new_attributes.items():
attr = prim.GetAttribute(attr_name)
if attr.IsValid():
attr.Set(attr_value)
else:
print(f"Warning: Attribute '{attr_name}' not found")
# --------------------------
# TAKE A PICTURE
# --------------------------
def to_numpy(data):
if data is None:
return None
# 1. Check for Warp Array (wp.array) - Specific to Isaac Sim 5.1+
if hasattr(data, "numpy") and callable(data.numpy):
return data.numpy()
# 2. Check for PyTorch Tensor
if hasattr(data, "cpu"):
return data.cpu().numpy()
# 3. Check for CuPy Array
if hasattr(data, "get"):
return data.get()
# 4. Fallback for standard lists/already NumPy
try:
return np.array(data)
except Exception:
return data
#Leaving in all the annotators I used in case this matters
class MultiDataWriterInstanceSeg(rep.Writer):
def __init__(self, output_dir):
self._output_dir = output_dir
self.version = "1.0.0"
self._rgb_dir = os.path.join(output_dir, "images_raw")
self._seg_dir = os.path.join(output_dir, "images_raw_seg")
self._frame_id = 1
self.success_count = 0
self.last_avg = None
self.get_gray = False
self.last_write_successful = False
for path in [self._rgb_dir, self._seg_dir]:
if not os.path.exists(path):
os.makedirs(path)
self.rgb_annot = rep.AnnotatorRegistry.get_annotator("rgb", device="cuda")
self.seg_annot = rep.AnnotatorRegistry.get_annotator("semantic_segmentation", init_params={"colorize": True}, device="cuda")
self.bbox_annot = rep.AnnotatorRegistry.get_annotator("bounding_box_2d_tight")
self.bbox3d_annot = rep.AnnotatorRegistry.get_annotator("bounding_box_3d")
self.camera_annot = rep.AnnotatorRegistry.get_annotator("camera_params")
self.annotators = [
self.rgb_annot,
self.seg_annot,
self.bbox_annot,
self.bbox3d_annot,
self.camera_annot
]
def write(self, data):
self.last_write_successful = False
# 1. Process RGB Data
rgb_pixels = None
rgb_pixels = to_numpy(data.get("rgb"))
if rgb_pixels is None:
print(" DEBUG: RGB Image is None.")
return 4
if not np.any(rgb_pixels):
print(" DEBUG: RGBA layers all 0.")
return 3
if not np.any(rgb_pixels[..., :3]):
print(" DEBUG: RGB layers all 0. Transparency layer non-zero.")
return 2
self.last_avg = np.mean(rgb_pixels[..., :3])
self.last_write_successful = True
return 1
def write_metadata(self, *args, **kwargs):
pass
def setup_picture_saver(camera_path):
render_product = rep.create.render_product(camera_path, (1920, 1440))
simulation_app.update()
writer = MultiDataWriterInstanceSeg("output")
writer.rgb_annot.attach(render_product)
writer.seg_annot.attach(render_product)
writer.bbox_annot.attach(render_product)
writer.bbox3d_annot.attach(render_product)
writer.camera_annot.attach(render_product)
rep.orchestrator.set_capture_on_play(False)
return writer, render_product
# --------------------------
# RUN SIMULATION AND SHOW STAGE
# ------------------------
world, stage = general_setup()
create_gray_card()
aperture_w, aperture_h, focal_len, max_height = get_camera_params()
setup_camera(aperture_w, aperture_h, focal_len, max_height, camera_path)
# Min height
min_height = max_height * percent_variance
# Random height between min and max
camera_height = random.uniform(min_height, max_height)
set_camera_height(max_height, camera_path)
rect_light_prim = create_rect_light(max_height)
sphere_light_prim = create_sphere_light(max_height)
cylinder_x1_light_prim = create_cylinder_x1_light(max_height)
cylinder_x2_light_prim = create_cylinder_x2_light(max_height)
cylinder_y1_light_prim = create_cylinder_y1_light(max_height)
cylinder_y2_light_prim = create_cylinder_y2_light(max_height)
writer, render_product = setup_picture_saver(camera_path)
print("Adding physics callback")
world.add_physics_callback("multiple_rotation_constrainer", constrain_multiple_rotations_callback)
print("Starting sim loop")
for i in range(10):
if not simulation_app.is_running():
break
world.stop()
# Min height (80% of plane visible)
min_height = max_height * percent_variance
# Random height between min and max
camera_height = random.uniform(min_height, max_height)
set_camera_height(camera_height, camera_path)
new_color = Gf.Vec3f(1.0, 1.0, 1.0)
new_intensity = 5000
new_attributes={
"inputs:intensity": 0 # Intensity of the light
}
set_light_attributes(rect_light_prim, new_attributes)
set_light_attributes(sphere_light_prim, new_attributes)
set_light_attributes(cylinder_x1_light_prim, new_attributes)
set_light_attributes(cylinder_x2_light_prim, new_attributes)
set_light_attributes(cylinder_y1_light_prim, new_attributes)
set_light_attributes(cylinder_y2_light_prim, new_attributes)
choose_light = random.randint(1, 4)
x_bounds = bg_x
y_bounds = bg_y
height_var = random.uniform(0.5, 1)
if choose_light == 1: #Rect
new_position = np.array([random.uniform(-x_bounds*2, x_bounds*2), random.uniform(-y_bounds*2, y_bounds*2), max_height*height_var])
new_attributes={
"inputs:width": bg_x * random.uniform(0.3, 2), # Width of the rectangular light
"inputs:height": bg_y * random.uniform(0.3, 2), # Height of the rectangular light
"inputs:intensity": new_intensity, # Intensity of the light
"inputs:color": new_color, # Color of the light (RGB)
"inputs:enableColorTemperature": False, # Disable color temperature for direct color setting
}
set_light_params(rect_light_prim, new_position, new_attributes)
if choose_light == 2: #Sphere
new_position = np.array([random.uniform(-x_bounds*2, x_bounds*2), random.uniform(-y_bounds*2, y_bounds*2), max_height*height_var])
new_attributes={
"inputs:radius": max(bg_x,bg_y) * random.uniform(0.3,1)/2, # Radius of the spherical light
"inputs:intensity": new_intensity, # Intensity of the light
"inputs:color": new_color, # Color of the light (RGB)
"inputs:enableColorTemperature": False, # Disable color temperature for direct color setting
}
set_light_params(sphere_light_prim, new_position, new_attributes)
if choose_light == 3: #CylinderX
length = bg_x * random.uniform(0.25, 2)
radius = random.uniform(0.005,0.03)
x_pos = random.uniform(-(bg_x - length)/2, (bg_x - length)/2)
y1_pos = random.uniform(0.25*bg_y/2, 1*bg_y/2)
y2_pos = -y1_pos
new_position = np.array([x_pos, y1_pos, max_height*height_var])
new_attributes={
"inputs:radius": radius, # Radius of the cylindrical light
"inputs:length": length, # Radius of the cylindrical light
"inputs:intensity": new_intensity, # Intensity of the light
"inputs:color": new_color, # Color of the light (RGB)
"inputs:enableColorTemperature": False, # Disable color temperature for direct color setting
}
set_light_params(cylinder_x1_light_prim, new_position, new_attributes)
new_position = np.array([x_pos, y2_pos, max_height*height_var])
set_light_params(cylinder_x2_light_prim, new_position, new_attributes)
if choose_light == 4: #CylinderY
length = bg_y * random.uniform(0.25, 2)
radius = random.uniform(0.005,0.03)
y_pos = random.uniform(-(bg_y - length)/2, (bg_y - length)/2)
x1_pos = random.uniform(0.25*bg_x/2, 1*bg_x/2)
x2_pos = -x1_pos
new_position = np.array([x1_pos, y_pos, max_height*height_var])
new_attributes={
"inputs:radius": radius, # Radius of the cylindrical light
"inputs:length": length, # Radius of the cylindrical light
"inputs:intensity": new_intensity, # Intensity of the light
"inputs:color": new_color, # Color of the light (RGB)
"inputs:enableColorTemperature": False, # Disable color temperature for direct color setting
}
set_light_params(cylinder_y1_light_prim, new_position, new_attributes)
new_position = np.array([x2_pos, y_pos, max_height*height_var])
set_light_params(cylinder_y2_light_prim, new_position, new_attributes)
for _ in range(10):
world.step(render=True)
# Calibrate exposure/intensity
graycard_path = "/World/GrayCardMesh"
move_prim(graycard_path, 0, 0, 0)
target_average = random.randint(88, 168)
tolerance = 5
max_iterations = 100
print("")
print(f"Image {i+1}")
print(" Calibrating gray card")
for j in range(max_iterations):
print(f" Iteration {j+1}")
success = False
attempts = 0
writer.get_gray = True
writer.last_avg = None
writer.last_write_successful = False
rgb_data = None
while not writer.last_write_successful and attempts < 1000:
for _ in range(10):
world.step(render=True)
rep.orchestrator.step(rt_subframes=16, delta_time=0.1)#, pause_timeline=False)
rep.orchestrator.wait_until_complete()
combined_data = {
"rgb": writer.rgb_annot.get_data(),
"semantic_segmentation": writer.seg_annot.get_data(),
"bounding_box_2d_tight": writer.bbox_annot.get_data(),
"bounding_box_3d": writer.bbox3d_annot.get_data(),
"camera_params": writer.camera_annot.get_data()
}
# 5. Call your write function manually
result = writer.write(combined_data)
if not writer.last_write_successful:
attempts += 1
if result == 2:
new_intensity = new_intensity + 1000
new_attributes={
"inputs:intensity": new_intensity # Intensity of the light
}
if choose_light == 1:
set_light_attributes(rect_light_prim, new_attributes)
if choose_light == 2:
set_light_attributes(sphere_light_prim, new_attributes)
if choose_light == 3:
set_light_attributes(cylinder_x1_light_prim, new_attributes)
set_light_attributes(cylinder_x2_light_prim, new_attributes)
if choose_light == 4:
set_light_attributes(cylinder_y1_light_prim, new_attributes)
set_light_attributes(cylinder_y2_light_prim, new_attributes)
print(f" Gray attempts: {attempts}")
average_value = writer.last_avg
# Check if we are within the target range
if abs(average_value - target_average) <= tolerance:
break
new_intensity = new_intensity * (target_average / average_value) * (target_average / average_value)
new_attributes={
"inputs:intensity": new_intensity, # Intensity of the light
}
if choose_light == 1:
set_light_attributes(rect_light_prim, new_attributes)
if choose_light == 2:
set_light_attributes(sphere_light_prim, new_attributes)
if choose_light == 3:
set_light_attributes(cylinder_x1_light_prim, new_attributes)
set_light_attributes(cylinder_x2_light_prim, new_attributes)
if choose_light == 4:
set_light_attributes(cylinder_y1_light_prim, new_attributes)
set_light_attributes(cylinder_y2_light_prim, new_attributes)
writer.get_gray = False
graycard_path = "/World/GrayCardMesh"
move_prim(graycard_path, 0, 0, -0.1)
simulation_app.close()
And here’s an example output I get:
Adding physics callback
Starting sim loop
2026-02-26T02:23:33Z [25,346ms] [Warning] [omni.syntheticdata.plugin] SdPostRenderVarToHost : invalid input resource for renderVar SemanticBoundingBox2DExtentTightSD
2026-02-26T02:23:33Z [25,498ms] [Warning] [omni.replicator.core.plugin] OgnSemanticOcclusionReduction: no semantics in the scene.
Module omni.replicator.core.ogn.python.impl.nodes.OgnSemanticSegmentation ae958f7 load on device 'cuda:0' took 2.35 ms (cached)
Image 1
Calibrating gray card
Iteration 1
Iteration 2
Iteration 3
Iteration 4
Image 2
Calibrating gray card
Iteration 1
Iteration 2
Iteration 3
Iteration 4
Iteration 5
Iteration 6
Image 3
Calibrating gray card
Iteration 1
Iteration 2
Iteration 3
Iteration 4
DEBUG: RGBA layers all 0.
Gray attempts: 1
Iteration 5
Image 4
Calibrating gray card
Iteration 1
Iteration 2
Iteration 3
Image 5
Calibrating gray card
Iteration 1
Iteration 2
Iteration 3
Iteration 4
DEBUG: RGBA layers all 0.
Gray attempts: 1
Iteration 5
Iteration 6
Image 6
Calibrating gray card
Iteration 1
Iteration 2
Iteration 3
DEBUG: RGBA layers all 0.
Gray attempts: 1
Iteration 4
Image 7
Calibrating gray card
Iteration 1
Iteration 2
Iteration 3
Image 8
Calibrating gray card
Iteration 1
Iteration 2
Image 9
Calibrating gray card
Iteration 1
Iteration 2
Iteration 3
Iteration 4
Iteration 5
Image 10
Calibrating gray card
Iteration 1
Iteration 2
Iteration 3
Iteration 4
Iteration 5
Iteration 6
Iteration 7
[109.099s] Simulation App Shutting Down