Hi! I made a script node in which I am able to get images from the camera in my simulation and then pass those images through a pre-trained model and get inference from it. I followed this video for making the script [https://www.youtube.com/watch?v=OdbRQnz2Btc](https://Isaac Sim/Robotics Weekly Livestream: Pytorch and Omnigraph)
What I want to ask is that currently the inference only works if I select the camera viewport otherwise, it doesn’t get any images. Is it possible to make it work even if the camera viewport is inactive or maybe allow multiple viewports to be active at the same time?
This is the script i am using
import torch
import torchvision
import omni.kit.viewport.utility as viewport_utils
from functools import partial
from PIL import Image
from omni.kit.widget.viewport.capture import ByteCapture
import ctypes
import numpy as np
def on_capture_completed(db, buffer, buffer_size , width , height, format):
# print("buffer", buffer)
# print("buffer_size", buffer_size)
# print("width", width)
# print("heigth", height)
# print("format", format)
ctypes.pythonapi.PyCapsule_GetPointer.restype = ctypes.POINTER(ctypes.c_byte*buffer_size)
ctypes.pythonapi.PyCapsule_GetPointer.argtypes = [ctypes.py_object,ctypes.c_char_p]
content = ctypes.pythonapi.PyCapsule_GetPointer(buffer, None)
img_content = content.contents
db.internal_state.img_buffer = Image.frombytes("RGBA", (width,height), img_content)
def setup(db: og.Database):
viewport_api = viewport_utils.get_active_viewport()
if "model" not in db.internal_state.__dict__:
# sys.path.append("C:/Users/General/Downloads/cookie_conveyor_emaad/yolo_v3")
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True, progress=True, num_classes=91, weights_backbone='ResNet50_Weights.DEFAULT').to(device)
img_buffer = None
model.eval()
db.internal_state.__dict__["models"] = model
db.internal_state.__dict__["device"] = device
db.internal_state.__dict__["img_buffer"] = img_buffer
capture = viewport_api.schedule_capture(ByteCapture(partial(on_capture_completed,db), aov_name = 'LdrColor'))
def compute(db: og.Database):
models = db.internal_state.models
device = db.internal_state.device
COCO_INSTANCE_CATEGORY_NAMES = [
'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A',
'N/A', 'truck', 'boat', 'N/A', 'N/A', 'N/A', 'N/A',
'N/A', 'N/A', 'N/A', 'cat', 'dog', 'horse', 'sheep', 'cow',
'elephant', 'bear', 'zebra', 'giraffe', 'N/A', 'N/A', 'umbrella', 'N/A', 'N/A',
'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A',
'N/A', 'baseball bat', 'N/A', 'N/A', 'N/A', 'N/A',
'bottle', 'N/A', 'N/A', 'cup', 'N/A', 'N/A', 'N/A', 'N/A',
'banana', 'apple', 'sandwich', 'orange', 'N/A', 'carrot', 'hot dog', 'pizza',
'donut', 'cake', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A',
'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A',
'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'book',
'clock', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A']
viewport_api = viewport_utils.get_active_viewport()
capture = viewport_api.schedule_capture(ByteCapture(partial(on_capture_completed,db), aov_name = 'LdrColor'))
image = db.internal_state.img_buffer
if image is not None:
image = np.array(image.convert("RGB"))
transformed_img = torchvision.transforms.transforms.ToTensor()(torchvision.transforms.ToPILImage()(image))
result = models([transformed_img.to(device)])
label = COCO_INSTANCE_CATEGORY_NAMES[result[0]['labels'][0]]
print(label)
db.outputs.class_name = label
else:
print("No image gotten!")
def cleanup(db: og.Database):
pass
Any help would be much appreciated.