Randomizer based Replicator code gets slower every frame

The following code which is a minor tweak of this demo from the docs gets slower to render with each frame. It’s much slower when run as an extension vs. headless, but the phenomenon is the same. Per frame times go from sub second to something like 4 minutes per frame after just 1000 frames.

Anyone else seeing this or have any bright ideas on how to fix, other than restarting every 100 frames or so?

import omni.replicator.core as rep

PS C:\Users\Dave Holden\AppData\Local\ov\pkg\code-2022.3.3> .\omni.code.bat --no-window --/omni/replicator/script=‘C:\Users\Dave Holden\Documents\DavePythonTest\headlessTest3.py’

def on_click():
# print(“Made it into the script”)
print(“Made it into the script”)
with rep.new_layer():
# Define paths for the character, the props, the environment, and the surface where the assets will be scattered in.
# WORKER = ‘omniverse://localhost/NVIDIA/Assets/Characters/Reallusion/Worker/Worker.usd’
WORKER = ‘C:/Users/Public/cgiAssets/Drill2.obj’
# PROPS = ‘omniverse://localhost/NVIDIA/Assets/Vegetation/Shrub’
PROPS = ‘C:/Users/Public/cgiAssets/’
ENVS = ‘omniverse://localhost/NVIDIA/Assets/Scenes/Templates/Outdoor/Puddles.usd’
SURFACE = ‘omniverse://localhost/NVIDIA/Assets/Scenes/Templates/Basic/display_riser.usd’
# SURFACE = ‘omniverse://localhost/NVIDIA/Assets/Scenes/Templates/Basic/abandoned_parking_lot.usd’

    # Define randomizer function for Base assets. This randomization includes placement and rotation of the assets on the surface.
    def env_props(size=17):
        instances = rep.randomizer.instantiate(rep.utils.get_usd_files(PROPS, recursive=True), size=size, mode='point_instance')
        with instances:
            rep.modify.pose(
                position=rep.distribution.uniform((-500, 0, -500), (500, 300, 500)),
                rotation=rep.distribution.uniform((-90,-180, -180), (-90, 180, 180)),
            )
        return instances.node

    def worker():
        worker = rep.create.from_usd(WORKER, semantics=[('class', 'worker')])

        with worker:
            rep.modify.pose(
                position=rep.distribution.uniform((0, 0, 0), (500, 500, 500)),
                rotation=rep.distribution.uniform((-180,-180, -180), (180, 180, 180)),
            )
        return worker

    def coney():
        distScale = 500
        cones = rep.create.cone(
                    position=rep.distribution.uniform((-distScale,0,-distScale),(distScale,distScale,distScale)), 
                    rotation=rep.distribution.uniform((-180,-180, -180), (180, 180, 180)),count=20)
        with cones:
            # rep.randomizer.color(colors=rep.distribution.uniform((0, 0, 0), (1, 1, 1)))
            rep.randomizer.texture(textures=[
                'omniverse://localhost/NVIDIA/Materials/vMaterials_2/Ground/textures/aggregate_exposed_diff.jpg',
                'omniverse://localhost/NVIDIA/Materials/vMaterials_2/Ground/textures/gravel_track_ballast_diff.jpg',
                'omniverse://localhost/NVIDIA/Materials/vMaterials_2/Ground/textures/gravel_track_ballast_multi_R_rough_G_ao.jpg',
                'omniverse://localhost/NVIDIA/Materials/vMaterials_2/Ground/textures/rough_gravel_rough.jpg'
                ])
        return cones
    
    def random_textured_plane():
        plane = rep.create.plane(
            rep.randomizer.scale(scale=rep.distribution.uniform((100, 0, 100), (1000, 0, 1000))), 
            rep.randomizer.texture(textures=[
                'omniverse://localhost/NVIDIA/Materials/vMaterials_2/Ground/textures/aggregate_exposed_diff.jpg',
                'omniverse://localhost/NVIDIA/Materials/vMaterials_2/Ground/textures/gravel_track_ballast_diff.jpg',
                'omniverse://localhost/NVIDIA/Materials/vMaterials_2/Ground/textures/gravel_track_ballast_multi_R_rough_G_ao.jpg',
                'omniverse://localhost/NVIDIA/Materials/vMaterials_2/Ground/textures/rough_gravel_rough.jpg'
            ])
        )   
        return plane
    
    def spheres():
        distScale = 500
        mycount = 20
        # mats = rep.create.material_omnipbr(diffuse=rep.distribution.uniform((0,0,0), (1,1,1)), count=mycount)
        mat1 = rep.create.material_omnipbr(count=mycount,
            diffuse=rep.distribution.uniform((0, 0, 0), (1, 1, 1)),
            roughness=rep.distribution.uniform(0, 1),
            metallic=rep.distribution.choice([0, 1]),
            emissive_color=rep.distribution.uniform((0, 0, 0.5), (0, 0, 1)),
            emissive_intensity=rep.distribution.uniform(0, 1000),
            )
        spheres = rep.create.sphere(position=rep.distribution.uniform((-distScale,0,-distScale),(distScale,distScale,distScale)), count=mycount)
        with spheres:
            # rep.randomizer.color(colors=rep.distribution.uniform((0, 0, 0), (1, 1, 1)))
            rep.randomizer.materials(materials=mat1)
        return spheres

    def dome_lights():
        lights = rep.create.light(
            light_type="Dome",
            rotation= (270,0,0),
            texture=rep.distribution.choice([
                'omniverse://localhost/NVIDIA/Assets/Skies/Cloudy/champagne_castle_1_4k.hdr',
                'omniverse://localhost/NVIDIA/Assets/Skies/Clear/evening_road_01_4k.hdr',
                'omniverse://localhost/NVIDIA/Assets/Skies/Clear/mealie_road_4k.hdr',
                'omniverse://localhost/NVIDIA/Assets/Skies/Clear/qwantani_4k.hdr'
                ])
            )
        return lights.node


    # Register randomization
    rep.randomizer.register(env_props)
    rep.randomizer.register(worker)
    rep.randomizer.register(coney)
    rep.randomizer.register(spheres)
    rep.randomizer.register(dome_lights)
    # rep.randomizer.register(random_textured_plane)

    # Setup the static elements
    env = rep.create.from_usd(ENVS)
    surface = rep.create.from_usd(SURFACE)

    # Setup camera and attach it to render product
    camera = rep.create.camera(
        focus_distance=800,
        f_stop=0.5
    )
    render_product = rep.create.render_product(camera, resolution=(224,224))

    # Initialize and attach writer
    writer = rep.WriterRegistry.get("BasicWriter")
    output_dir = r"C:\Users\Dave Holden\Documents\ReplicatorScripts\assets5"
    writer.initialize(output_dir=output_dir, rgb=True, bounding_box_2d_tight=True) # , semantic_segmentation=True)
    writer.attach([render_product])
    # import gc

    # with rep.trigger.on_time(num=100, interval=1, rt_subframes=4):
    with rep.trigger.on_frame(num_frames=25):
        rep.randomizer.env_props(15)
        rep.randomizer.worker()
        rep.randomizer.coney()
        rep.randomizer.spheres()
        rep.randomizer.dome_lights()
        # rep.randomizer.random_textured_plane()

        with camera:
            rep.modify.pose(position=rep.distribution.uniform((-500, 200, 1000), (500, 500, 1500)), look_at=surface)

    rep.orchestrator.run()  # Run the randomization
        # gc.collect()  # Force garbage collection

on_click()

1 Like

This is essentially the demo from the code but pointing to local assets so it’s fast.

Only changes (except camera image is smaller too)
render_product = rep.create.render_product(camera, resolution=(224, 224))

# WORKER = 'omniverse://localhost/NVIDIA/Assets/Characters/Reallusion/Worker/Worker.usd'
WORKER = 'C:/Users/Public/cgiAssets/Drill2.obj'
# PROPS = 'omniverse://localhost/NVIDIA/Assets/Vegetation/Shrub'
PROPS =   'C:/Users/Public/cgiAssets/'
ENVS = 'omniverse://localhost/NVIDIA/Assets/Scenes/Templates/Outdoor/Puddles.usd'
SURFACE = 'omniverse://localhost/NVIDIA/Assets/Scenes/Templates/Basic/display_riser.usd'

BTW, I timed this on Ubuntu and Windows 11 and I see the same pattern.

===
More fun things to know, even if you make an extension that generates 10 frames and keep running it from the Replicator menu, it STILL takes more time each frame if you run it several times in a row. I thought it would garbage collect or something in between, but nope.

image

Help me Obi Wan…

Also
with rep.trigger.on_time(num=10):
has same growth pattern as
with rep.trigger.on_frame(num_frames=10):

And just for good measure, here’s NVIDIA’s recent fruit demo timing. I haven’t changed anything in the code except the output directory. Otherwise it’s verbatim from github.

Here is perhaps a better way to get the point across, the time to render grows exponentially with the number of images, where everyone expects it to be linear.

The only changes I made to the fruit demo was output image size, output directory and number of images.

Hello @holdendp and thank you so much for bringing this to our attention. We’ve identified the cause and will have a fix implemented in the next release. Meanwhile, a workaround is to use mode="reference" instead of mode="point_instance" within the instantiate() call. This comes at a performance penalty vs. the fixed point_instance version but will avoid the bug that you’ve uncovered. In testing this version, I discovered that images are now rendering before all materials load. Use rt_subframes=3 in the trigger to avoid this issue (effectively renders 3 frames per iteration). See code snippet below:

import os
import csv
import time
import omni.replicator.core as rep

OUT_PATH = "/my/output/path"

class MyWriter(rep.Writer):
    def __init__(self):
        self.annotators = ["rgb", "bounding_box_2d_tight"]
        self.last_time = None
        self.csv_path = os.path.join(OUT_PATH, "deltas.csv")
        self.frame_id = 0
        self.backend = rep.BackendDispatch({"paths": {"out_dir": OUT_PATH}})
        with open(self.csv_path, "w", newline="") as file:
            writer = csv.writer(file)
            writer.writerow(["frame", "delta"])

    def write(self, data):
        self.backend.write_image(f"frame_{self.frame_id}.png", data["rgb"])
        cur_time = time.time()

        # Write time delta to csv
        if self.last_time is not None:
            with open(self.csv_path, 'a') as f:
                writer = csv.writer(f)
                writer.writerow([self.frame_id, cur_time - self.last_time])

        self.last_time = cur_time
        self.frame_id += 1

with rep.new_layer():
    # Define paths for the character, the props, the environment and the surface where the assets will be scattered in.

    WORKER = 'omniverse://localhost/NVIDIA/Assets/Characters/Reallusion/Worker/Worker.usd'
    PROPS = 'omniverse://localhost/NVIDIA/Assets/Vegetation/Shrub'
    ENVS = 'omniverse://localhost/NVIDIA/Assets/Scenes/Templates/Outdoor/Puddles.usd'
    SURFACE = 'omniverse://localhost/NVIDIA/Assets/Scenes/Templates/Basic/display_riser.usd'

    # Define randomizer function for Base assets. This randomization includes placement and rotation of the assets on the surface.
    def env_props(size=50):
        instances = rep.randomizer.instantiate(rep.utils.get_usd_files(PROPS), size=size, mode='reference')
        with instances:
            rep.modify.pose(
                position=rep.distribution.uniform((-500, 0, -500), (500, 0, 500)),
                rotation=rep.distribution.uniform((-90,-180, 0), (-90, 180, 0)),
            )
        return instances.node

    def worker():
        worker = rep.create.from_usd(WORKER, semantics=[('class', 'worker')])

        with worker:
            rep.modify.pose(
                position=rep.distribution.uniform((-500, 0, -500), (500, 0, 500)),
                rotation=rep.distribution.uniform((-90,-45, 0), (-90, 45, 0)),
            )
        return worker

    # Register randomization
    rep.randomizer.register(env_props)
    rep.randomizer.register(worker)

    # Setup the static elements
    env = rep.create.from_usd(ENVS)
    surface = rep.create.from_usd(SURFACE)

    # Setup camera and attach it to render product
    camera = rep.create.camera(
        focus_distance=800,
        f_stop=0.5
    )
    render_product = rep.create.render_product(camera, resolution=(899, 899))

    # Initialize and attach writer
    writer = MyWriter()
    writer.initialize()
    writer.attach([render_product])

    with rep.trigger.on_frame(num_frames=5, rt_subframes=3):
        rep.randomizer.env_props(75)
        rep.randomizer.worker()
        with camera:
            rep.modify.pose(position=rep.distribution.uniform((-500, 200, 1000), (500, 500, 1500)), look_at=surface)```

Please let us know if this is sufficient to unblock you. If you absolutely need to use point instancers, let us know and I can detail how to implement a fix on your local build.
1 Like

Cool, thanks. I ran your code with 200 frames and it is consistent time per frame now. Thanks for fixing this and I’m glad my efforts could help you.

Also, thanks for fixing the materials not rendering issue.

-Dave

While the exact code you provided does have consistent render times, the proposed fix does not fix the issue in general. e.g. when applied to the fruit crate example I still see the issue.

Meanwhile, a workaround is to use mode="reference" instead of mode="point_instance" within the instantiate() call.

fyi

Thanks for the update @holdendp . Could you please provide some details regardign the Fruit Example script you’re using?

It’s NVIDIA’s code I believe:

I’m just running this, as is on GitHub and then with the suggested workaround

Meanwhile, a workaround is to use mode="reference" instead of mode="point_instance" within the instantiate() call.

They both get slower with each frame as of a day or two ago at least.

Let me know if questions.

I am also having the same experience as mentioned above. Both mode=“reference” and mode="mode=“point_instance” are causing a slowing of the rendering. I have to say, that I was able to reduce the effect, making the instantiate() call before calling rep.trigger.on_frame(), but it still is very noticeable.

image

After doing some further testing I can say that calling instantiate outside of the for loop fixes the issue for me:

1 Like

Thank you @julian.grimm and @holdendp. With your help, we were able to locate the root cause of this slowdown and implement a fix. Please look for it in the next upcoming IsaacSim release! If this remains a blocker, please let us know and we can provide steps to patch the problematic files in existing builds.

1 Like

Thank you for taking a look at this issue. I am currently using Replicator in Omniverse Code. Will this fix be already implemented in the upcoming release?

1 Like

Unfortunately the fix will not make it for the new Code release. However if this is blocking progress in your work, please let us know and we can walk through steps to address the issue based on your use case.

1 Like

Thank you for providing help on this topic. The issue where the rendering slows down is currently not a problem for me because I do not have to render that many images at once. However I am curious if this bug is related to the problem I mentioned in here:

Maybe there is a better way of achieving the following:

  1. Instantiate a bunch of Parts (Given by USD-Path)
  2. Modify the semantics of the parts (1 semantic class for each USD-Path)
  3. Randomize the rotation around the z-axis of the part
  4. Scatter it over a plane

Currently my implementation looks like this:

# region Randomizer methods
# Randomize Part Position and Rotatation
def random_Parts():
    # Retrieve the prims of the instances
    instances = rep.get.prims(
        semantics=[('class', 'cube'), ('class', 'capsule'), ('class', 'cone'), ('class', 'cylinder'), ('class', 'sphere'), ('class', 'torus')]
    )

    with instances:
        # Randomize the rotation of the parts
        rep.randomizer.rotation(
            min_angle=(0, 0, -180),
            max_angle=(1, 1, 180)
        )

        # Scatter it across the plane
        rep.randomizer.scatter_2d(
            surface_prims=spawnplane,
            check_for_collisions=True,
        )
    return instances.node

# Instantiate a random number of parts
def instantiate_Parts(file_path, classes, sample_size):
    # Instantiates a number of parts (size specifies the number of instances per part)
    # The distribution of the instances over all is random
    instances = rep.randomizer.instantiate(
        paths=file_path,
        size=sample_size,
        mode='reference'
    )

    with instances:
        # Label each part
        rep.modify.semantics([('class', classes)])

        # Randomize the parts material
        rep.randomizer.materials(
            materials=rep.get.material(path_pattern="/Looks/Parts/*")
        )

# Randomize the floor material
def random_Floor_Material():
    floor_material = rep.randomizer.materials(
        materials=rep.get.material(path_pattern="/Looks/Floor/*"),
        input_prims=floor
    )
    return floor_material.node

# Randomize Dome Light
def dome_Light():
    lights = rep.create.light(
        light_type="Dome",
        position=(0, 0, 0),
        rotation=rep.distribution.uniform((0, 0, -180), (0, 0, 180)),
        scale=(1, 1, 1),
        name='HDRI',
        texture=rep.distribution.choice([
            'https://omniverse-content-production.s3.us-west-2.amazonaws.com/Assets/Skies/2022_1/Skies/Indoor/autoshop_01.hdr',
            'https://omniverse-content-production.s3.us-west-2.amazonaws.com/Assets/Skies/2022_1/Skies/Indoor/small_hangar_01.hdr',
            'http://omniverse-content-production.s3-us-west-2.amazonaws.com/Assets/Skies/Indoor/ZetoCGcom_ExhibitionHall_Interior1.hdr',
            'https://omniverse-content-production.s3.us-west-2.amazonaws.com/Assets/Skies/2022_1/Skies/Indoor/adams_place_bridge.hdr'
            ])
        )
    return lights.node

# Randomize the Focus Distance of the camera
def random_Focus_Distance(camera, MinFocusDistance, MaxFocusDistance):
    with camera:
        rep.modify.attribute(
            name='focusDistance',
            value=rep.distribution.uniform(MinFocusDistance, MaxFocusDistance)
        )
    return camera.node

# Register Randomizers
rep.randomizer.register(random_Parts)
rep.randomizer.register(instantiate_Parts)
rep.randomizer.register(random_Floor_Material)
rep.randomizer.register(dome_Light)
rep.randomizer.register(random_Focus_Distance)
# endregion
  # Instatiate all the parts
  for classes, path in PARTS.items():
      rep.randomizer.instantiate_Parts(path, classes, 3)
  
  # Trigger the randomizer at each frame
  with rep.trigger.on_frame(num_frames=num):
      rep.randomizer.random_Parts()
      rep.randomizer.random_Floor_Material()
      rep.randomizer.dome_Light()
      rep.randomizer.random_Focus_Distance(topview, 1200, 1600)

Thank you for your efforts!
Julian

I worked around the slowdown by restarting my headless rendering util after n images. Ugly hack but good enough until a real fix arrives.