Note: For any Isaac Lab topics, please submit your topic to its GitHub repo (GitHub - isaac-sim/IsaacLab: Unified framework for robot learning built on NVIDIA Isaac Sim) following the instructions provided on Isaac Lab’s Contributing Guidelines (Contribution Guidelines — Isaac Lab Documentation).
Please provide all relevant details below before submitting your post. This will help the community provide more accurate and timely assistance. After submitting, you can check the appropriate boxes. Remember, you can always edit your post later to include additional information if needed.
Isaac Sim Version
4.5.0
4.2.0
4.1.0
4.0.0
4.5.0
2023.1.1
2023.1.0-hotfix.1
Other (please specify):
Operating System
Ubuntu 22.04
Ubuntu 20.04
Windows 11
Windows 10
Other (please specify):
GPU Information
- Model: NVIDIA RTX 5880 Ada Generation
- Driver Version: 12.4
Topic Description
Detailed Description
(Describe the issue in detail, including what you were trying to do, what you expected to happen, and what actually happened)
- we use headless mode
- we use replicator to obtain the bbox position information on image, but they do not match
- we use issacim to obtain the 3d positions of a prim and get the 2d coordinates in the image, but the position is not right
Steps to Reproduce
- main
# set scene
world, viewport_api, stage, camera = set_scene()
i = 0
while simulation_app.is_running():
# --- 器械１ ---
# 将随机产生的位移、角度设置到器械关节并进行渲染
art_1 = dc.get_articulation("/RLND_8_G3_0")
dc.wake_up_articulation(art_1)
joint_angles_1 = [np.array([j1, j2, j3, j4, j5,
j6, j7, j8, j9, j10,
j11, j12, j13])] # 设定随机值
dc.set_articulation_dof_position_targets(art_1, joint_angles_1)
# --- 器械2 ---
art_2 = dc.get_articulation("/RMBF_8_G3_0")
dc.wake_up_articulation(art_2)
joint_angles_2 = [np.array([j1, j2, j3, j4, j5,
j6, j7, j8, j9, j10,
j11, j12, j13])] # 再次设定随机值
dc.set_articulation_dof_position_targets(art_2, joint_angles_2)
# Multiple rendering for smooth motion effects
for j in range(60): # note
world.step(render=True)
# 获取渲染的图像
get_viewport_img(save_img_path, save_image_name, viewport_api,i)
# 获取第一把器械的bounding-box的信息从replicator里
get_bbox_2d_loose()
# 获取第一把器械的keypoints的2d位置信息从isaacsim里
get_keypoints_2d()
# 获取第二把器械的bounding-box的信息从replicator里
get_bbox_2d_loose()
# 获取第二把器械的keypoints的2d位置信息从isaacsim里
get_keypoints_2d()
# 把信息整合成labelme可读的json文件
send_info_to_json_file()
i += 1
- set scene
import omni.kit.commands
from omni.isaac.dynamic_control import _dynamic_control
from pxr import Sdf, Gf, UsdPhysics, UsdLux, PhysxSchema, UsdGeom
from omni.isaac.sensor import Camera
import omni.isaac.core.utils.numpy.rotations as rot_utils
def set_scene():
# Setting up URDF import configuration 设置导入配置
status, import_config = omni.kit.commands.execute("URDFCreateImportConfig")
import_config.merge_fixed_joints = False
import_config.convex_decomp = False #禁用凸分解
import_config.import_inertia_tensor = True
import_config.fix_base = False
import_config.distance_scale = 100
status, stage_path = omni.kit.commands.execute(
"URDFParseAndImportFile",
urdf_path=current_work_dir + "/GEM-8mm-urdf-meshes/urdf/instr_RLND-8_v2.urdf", # surgical instrument 1
import_config=import_config,
)
status, stage_path = omni.kit.commands.execute(
"URDFParseAndImportFile",
urdf_path=current_work_dir + "/urdf/robot_G2_2_2_FleshColor.urdf", # SetupArmG
import_config=import_config,
)
status, stage_path = omni.kit.commands.execute(
"URDFParseAndImportFile",
urdf_path=current_work_dir + "/GEM-8mm-urdf-meshes/urdf/instr_RMBF-8_v2.urdf", # surgical instrument 2
import_config=import_config,
)
# Get stage handle
stage = omni.usd.get_context().get_stage()
# Add world
from omni.isaac.core import World
world = World()
# Enable physics 设置物理环境
scene = UsdPhysics.Scene.Define(stage, Sdf.Path("/physicsScene"))
# Set gravity 设置重力
scene.CreateGravityDirectionAttr().Set(Gf.Vec3f(0.0, 0.0, -1.0))
scene.CreateGravityMagnitudeAttr().Set(9.81)
# Set solver settings 配置物理求解器
PhysxSchema.PhysxSceneAPI.Apply(stage.GetPrimAtPath("/physicsScene"))
physxSceneAPI = PhysxSchema.PhysxSceneAPI.Get(stage, "/physicsScene")
physxSceneAPI.CreateEnableCCDAttr(True)#连续碰撞
physxSceneAPI.CreateEnableStabilizationAttr(True)
physxSceneAPI.CreateEnableGPUDynamicsAttr(False)
# Add ground plane
omni.kit.commands.execute(
"AddGroundPlaneCommand",
stage=stage,
planePath="/groundPlane",
axis="Z",
size=1500.0,
position=Gf.Vec3f(0, 0, 0),
color=Gf.Vec3f(0.5, 0.5, 0.5), # Note!
)
# set joint angle
dc = _dynamic_control.acquire_dynamic_control_interface()
def set_joint_angles(yaw_deg, pitch_deg):
# 获取articulation接口
dc = _dynamic_control.acquire_dynamic_control_interface()
articulation = dc.get_articulation("/SetupArmG2_2")
if articulation == _dynamic_control.INVALID_HANDLE:
print("Error: Articulation not found!")
return
# 新版API获取关节方式
dof_states = dc.get_articulation_dof_states(articulation, _dynamic_control.STATE_ALL)
# 设置关节角度（弧度）
joints_config = {
"A1234": np.deg2rad(yaw_deg), # Z轴旋转
"A5": np.deg2rad(pitch_deg) # Y轴旋转
}
# 更新DOF状态
for i in range(len(dof_states)):
joint_name = dc.get_dof_name(dc.get_articulation_dof(articulation, i))
if joint_name in joints_config:
dof_states[i]["pos"] = joints_config[joint_name]
# 应用新状态
dc.set_articulation_dof_states(articulation, dof_states, _dynamic_control.STATE_ALL)
# 示例：水平转向5° + 俯仰-5°
set_joint_angles(5, -5)
# Add camera
camera = Camera(
prim_path="/SetupArmG2_2/L5/Camera", # 相机在usd格式中的路径
translation=np.array([-0.65, 0, -0.8]),# 相机相对于父Prim的位移
frequency=60,#帧率
resolution=(1920, 1080),#分辨率
orientation=rot_utils.euler_angles_to_quats(np.array([0, 89, 129]), degrees=True)#欧拉角转四元数 plan b 旋转背景和相机
)
# Add some light and background images
# Flash world after add object
world.reset()
camera.initialize()
camera.set_resolution([1920, 1080])
# Get viewport api, bind camera and adjust resolution
from omni.kit.viewport.utility import get_viewport_from_window_name
viewport_api = get_viewport_from_window_name("Viewport")
viewport_api.set_active_camera("/SetupArmG2_2/L5/Camera") # 绑定场景所设置的相机(附加在机械臂上)
viewport_api.resolution=(1920, 1080)
world.step(render=True)
return world,viewport_api, stage, camera
- capture image
from omni.kit.viewport.utility import capture_viewport_to_file
import time
def get_viewport_img(save_path, img_name, viewport_api, i):
# Get image from specified viewport
imgae_path = "path/to/img.png"
capture_viewport_to_file(viewport_api, imgae_path)
# Wait a bit to ensure the file is saved
time.sleep(0.1)
- get bbox
from omni.isaac.core.utils.semantics import add_update_semantics
from omni.isaac.core.utils.extensions import enable_extension # isaacsim-v2023
enable_extension("omni.isaac.synthetic_utils") # isaacsim-v2023
from omni.isaac.synthetic_utils import SyntheticDataHelper # isaacsim-v2022
def get_bbox_2d_loose(labelname, selected_prims_bbox, camera_prim):
# 设置语义过滤器 labelname="器械名"
# selected_prims_bbox = [一个器械里的多个零部件]
for i in range(len(selected_prims_bbox)):
G_instr = stage.GetPrimAtPath(selected_prims_bbox[i])
add_update_semantics(G_instr, "object_"+str(i), type_label=labelname)
sd.SyntheticData.Get().set_instance_mapping_semantic_filter(f"{labelname}:*")
# camera prim是相机在isaacsim里面的prim路径
rp = rep.create.render_product(camera_prim, (1920, 1080))
bbox_2d_loose = rep.AnnotatorRegistry.get_annotator("bounding_box_2d_loose")
bbox_2d_loose.attach(rp)
rep.orchestrator.step()
bboxes_data = bbox_2d_loose.get_data() # 获得每个部位的bbox数据
rect_data = bbox_fuse(bboxes_data) # 将每个部位的数据合并成一个bbox
# 存储数据
data = []
# detach防止后续调用错误
try:
bbox_2d_loose.detach()
except Exception:
pass
return data
- get 2d coordinates
def get_keypoints_2d(stage, camera, selected_prims):
'''
Get the 3D positon information of concerned keypoints
input:
selected_prims = ["prim1", "prim2", "prim3"]
'''
# Get keypoints 3d pose
timeline = omni.timeline.get_timeline_interface() # Get the current timecode
timecode = timeline.get_current_time() * timeline.get_time_codes_per_seconds()
for i, s in enumerate(selected_prims):
curr_prim = stage.GetPrimAtPath(s) #获取场景中指定路径下的 USD 主体
#获取指定 USD 主体在世界坐标系中的变换矩阵
# pose = omni.usd.utils.get_world_transform_matrix(curr_prim, timecode) # isaacsim-v2022
pose = omni.usd.get_world_transform_matrix(curr_prim, timecode) # isaacsim-v2023
world_point = np.array([list(pose.ExtractTranslation())]) # keypoints-3d: 给定的世界变换矩阵 pose 中提取平移部分
image_coord = camera.get_image_coords_from_world_points(world_point) # keypoints-2d: 相机捕获
# 数据存储
data= []
return data
Error Messages
(If applicable, copy and paste any error messages you received)
Visualization not right
Screenshots or Videos
(If applicable, add screenshots or links to videos that demonstrate the issue)
Additional Information
What I’ve Tried
(Describe any troubleshooting steps you’ve already taken)
Related Issues
(If you’re aware of any related issues or forum posts, please link them here)
Additional Context
(Add any other context about the problem here)