Live inference using Yolov8

Hello.

I am trying to run live inference using Yolov8 inside Isaac sim’s script editor. I am able to run inference for a single frame but I want to run live inference. While doing so, I am not able to access the stage area or the render screen inside Isaac and I need to close the window of model inference.

I tried creating a new thread for mode inference but the same results. Attaching my code for reference. Please let me know if there’s any way to do this.

import omni
from omni.kit.viewport.utility import get_active_viewport
from omni.isaac.synthetic_utils import SyntheticDataHelper
import matplotlib.pyplot as plt
from ultralytics import YOLO
import threading

import cv2
import asyncio

sd_helper = SyntheticDataHelper()
viewport_api = get_active_viewport()
flag = True

model = YOLO('yolov8s.pt')



def run():
    sd_helper.initialize_async(["rgb", "depth"], viewport_api)
    flag = True
    while flag:
        gt = sd_helper.get_groundtruth(["rgb", "depth"], viewport_api, verify_sensor_init=False)
        rgb = gt["rgb"]
        depth = gt["depth"]
        #cv2.imwrite('/home/user/temp_isaac.png', rgb)
        print('mode_loaded')
        print(model)
        print('rgb: ', rgb)
        rgb = rgb[:,:,:3]
        results = model(rgb)
        print('forward_pass')
        annotated_frame = results[0].plot()
        print(annotated_frame)
        cv2.imwrite('/home/user/temp_isaac_result.png', annotated_frame)
        cv2.imshow('test', annotated_frame)
        if cv2.waitKey(33) == ord('a'):
            flag = False
            cv2.destroyAllWindows()

thread = threading.Thread(target=run)
thread.start()
thread.join()

Thanks,
Mayank

Hi there,

you can use this snippet in the script editor to access annotator data (e.g. rgb):

import asyncio
import omni.replicator.core as rep

NUM_FRAMES = 10

# Manually access the data from the RGB annotator
rep.orchestrator.set_capture_on_play(False)

async def get_rgb_data_async():
    render_product = rep.create.render_product("/OmniverseKit_Persp", (512, 512))
    rgb_annot = rep.AnnotatorRegistry.get_annotator("rgb")
    rgb_annot.attach(render_product)
    for i in range(NUM_FRAMES):
        await rep.orchestrator.step_async()
        rgb_data = rgb_annot.get_data()
        print(f"Frames {i}: RGB data shape: {rgb_data.shape}")
        # await asyncio.sleep(1.0)
        # for _ in range(10):
        #     await omni.kit.app.get_app().next_update_async()
    rgb_annot.detach()
    render_product.destroy()

asyncio.ensure_future(get_rgb_data_async())

Using asyncio.sleep() or omni.kit.app.get_app().next_update_async() you can control the frequency of your data update.