Hi, I encountered a problem with inserting external entities. When using NuRec’s gRPC to insert external dynamic objects, the insertion failed.
USDZ files generated using official data can be inserted normally, but USDZ files generated using my own sensor data cannot be inserted, although there is no error message.
environment
system:
Distributor ID: Ubuntu
Description: Ubuntu 20.04.6 LTS
Release: 20.04
Codename: focal
Graphics card driver version
NVIDIA-SMI 550.127.08 Driver Version: 550.127.08 CUDA Version: 12.4 |
|-----------------------------------------+------------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+========================+======================|
| 0 NVIDIA A100 80GB PCIe Off | 00000000:34:00.0 Off | 0 |
| N/A 77C P0 311W / 300W | 63391MiB / 81920MiB | 100% Default |
| | | Disabled
cuda version:
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2024 NVIDIA Corporation
Built on Tue_Feb_27_16:19:38_PST_2024
Cuda compilation tools, release 12.4, V12.4.99
Build cuda_12.4.r12.4/compiler.33961263_0
The following are the instructions or codes designed during the operation.
instructions
Generate NuRec Auxiliary Data
docker run --shm-size=2g -it --rm --gpus '"device=7"' \
-e NGC_API_KEY=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 \
--volume / NuRec/data/docker/dataset:/workdir/dataset \
--volume / NuRec/data/docker/output:/workdir/output \
nvcr.io/nvidia/nre/nre-tools-ga:latest \
--dataset-path=/workdir/dataset/yj/tj/my_custom_drive_001.json \
--output-dir=/workdir/output/yj/tj \
--store-meta \
--no-seg-logits \
--lidar-seg-camvis
Reconstruct
docker run --shm-size=64g --rm --gpus '"device=7"' \
-e NGC_API_KEY=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 \
--volume / NuRec/data/docker/dataset:/workdir/dataset \
--volume / NuRec/data/docker/output:/workdir/output \
nvcr.io/nvidia/nre/nre-ga:latest \
mode=train \
out_dir=/workdir/output/reconstruction/yj \
--config-name=configs/apps/prod/Hyperion-8.1/car2sim_6cam.yaml \
dataset.path=/workdir/dataset/yj/my_custom_drive_001.json \
dataset.camera_ids=[cam_back] \ # Change based on USDZ
dataset.lidar_ids=[lidar_back] \ # Change based on USDZ
dataset.aux_data=True
Use Asset Harvester Output in Reconstructions
docker run --shm-size=64g -it --rm --gpus '"device=7"' \
--net=host \
--privileged \
--volume / NuRec/data/docker/output:/workdir/output \
--volume / NuRec/data/docker/dataset:/workdir/dataset \
nvcr.io/nvidia/nre/nre-ga:latest \
export-external-assets \
--artifact-path /workdir/output/reconstruction/yj/SMZcLV7gPsq5ApwsySUeas/artifacts/last.usdz \
--external-assets-dir /workdir/dataset/harvester/customer/0 \
--output-edit-file /workdir/output/render-harvester/edit_assets.json \
--output-artifact-path /workdir/output/render-harvester/reconstruction-bj-2.usdz
External entity check（Both used the same external entity）
- Huggingface data generation
- Own sensor data generation
gRPC service startup command:
docker run --shm-size=64g -it --rm --gpus all \
-e CUDA_VISIBLE_DEVICES=7 \
--net=host \
--privileged \
--volume /NuRec/data/docker/output:/workdir/output \
nvcr.io/nvidia/nre/nre-ga:latest \
serve-grpc \
--host="0.0.0.0" \
--port=1001 \
--artifact-glob /workdir/output/render-harvester/reconstruction-n-pure.usdz \
--test-scenes-are-valid \
--enable-editing-actors \
--no-enable-nrend
API call Insert code
Summary
import asyncio
import grpc.aio
import numpy as np
from scipy.spatial.transform import Rotation as R
from nre.grpc.protos.common_pb2 import (
Empty, Pose, Vec3, Quat, Trajectory, PoseAtTime, AABB
)
# Import generated gRPC code
from nre.grpc.protos.sensorsim_pb2 import (
RGBRenderRequest,
PosePair,
ImageFormat,
EditAssetsRequest,
DynamicObjectTrack,
CameraSpec
)
from nre.grpc.protos.sensorsim_pb2_grpc import SensorsimServiceStub
def se3_to_grpc_pose(matrix_4x4):
"""
Convert a 4x4 SE3 matrix to a gRPC Pose object.
Args:
matrix_4x4 (np.ndarray): 4x4 transformation matrix (Rig to World or Camera to World)
Returns:
Pose: gRPC Pose object with vec (translation) and quat (rotation)
"""
translation = matrix_4x4[:3, 3]
rotation_matrix = matrix_4x4[:3, :3]
quat = R.from_matrix(rotation_matrix).as_quat() # (x, y, z, w)
return Pose(
vec=Vec3(x=float(translation[0]), y=float(translation[1]), z=float(translation[2])),
quat=Quat(x=float(quat[0]), y=float(quat[1]), z=float(quat[2]), w=float(quat[3]))
)
class TrajectoryController:
def __init__(self, host='10.102.1.70', port=1001):
self.host = host
self.port = port
self.channel = None
self.stub = None
async def connect(self):
self.channel = grpc.aio.insecure_channel(f'{self.host}:{self.port}')
self.stub = SensorsimServiceStub(self.channel)
print(f"Connected to SensorsimService at {self.host}:{self.port}")
async def get_scenes(self):
response = await self.stub.get_available_scenes(Empty())
return response.scene_ids
async def add_dynamic_vehicle(self, scene_id, track_id, trajectory_points, asset_id):
"""
Adds a dynamic vehicle (NPC) to the scene with a specific trajectory.
Args:
scene_id (str): Scene ID
track_id (str): Unique ID for this vehicle instance
trajectory_points (list): List of {'timestamp_us': ..., 'pose': 4x4 matrix}
asset_id (str): The model to use for the vehicle
"""
poses_at_time = []
for pt in trajectory_points:
poses_at_time.append(PoseAtTime(
pose=se3_to_grpc_pose(pt['pose']),
timestamp_us=pt['timestamp_us']
))
track = DynamicObjectTrack(
id=track_id,
semantic_class="automobile",
asset_id=asset_id,
trajectory=Trajectory(poses=poses_at_time),
object_size=AABB(size_x=4.5, size_y=1.8, size_z=1.5), # Approximate sedan size
)
request = EditAssetsRequest(
scene_id=scene_id,
insert=[track]
)
response = await self.stub.edit_assets(request)
if response.success:
print(f"Successfully added dynamic obj '{track_id}' to scene.")
else:
print(f"Failed to add dynamic obj: {response.message}")
return response
async def close(self):
if self.channel:
await self.channel.close()
def generate_linear_trajectory(start_pos, end_pos, duration_s, fps=100):
"""
Helper to generate a simple straight line trajectory.
"""
num_frames = int(duration_s * fps)
trajectory = []
for i in range(num_frames):
alpha = i / (num_frames - 1)
current_pos = start_pos + alpha * (end_pos - start_pos)
# Simple identity rotation (looking forward along X axis)
pose = np.eye(4)
pose[:3, 3] = current_pos
trajectory.append({
'timestamp_us': int(i * (1e6 / fps)),
'pose': pose
})
return trajectory
async def main():
controller = TrajectoryController()
await controller.connect()
scenes = await controller.get_scenes()
if not scenes:
print("No scenes available.")
return
scene_id = scenes[0]
# Add an NPC vehicle moving across the scene
npc_start = np.array([5, 5, 0])
npc_end = np.array([5, -5, 0])
npc_traj = generate_linear_trajectory(npc_start, npc_end, duration_s=2.0, fps=5)
await controller.add_dynamic_vehicle(scene_id, "0", npc_traj, '0')
await controller.close()
if __name__ == "__main__":
asyncio.run(main())
API call query code
Summary
import asyncio
async def query_all_available_objects(host='10.102.1.70', port=1001):
"""
Query all available external objects
"""
import grpc.aio
from nre.grpc.protos.sensorsim_pb2_grpc import SensorsimServiceStub
from nre.grpc.protos.common_pb2 import Empty
from nre.grpc.protos.sensorsim_pb2 import (
AvailableTrajectoriesRequest,
AvailableCamerasRequest,
AvailableDynamicObjectsRequest,
ExternalAssetObjectsRequest
)
channel = grpc.aio.insecure_channel(f'{host}:{port}')
stub = SensorsimServiceStub(channel)
try:
print("=" * 60)
print("All available external objects and services")
print("=" * 60)
# 1. Get available scenes
print("\n1. Available scenes")
print("-" * 40)
scenes_response = await stub.get_available_scenes(Empty())
if scenes_response.scene_ids:
for i, scene_id in enumerate(scenes_response.scene_ids):
print(f" [{i}] Scene ID: {scene_id}")
else:
print(" No available scenes")
if not scenes_response.scene_ids:
print("\nNo available scenes, cannot query other objects")
return
scene_id = scenes_response.scene_ids[0]
print(f"\nUsing scene: {scene_id} for detailed query")
# 2. Get available trajectories
print("\n2. Available trajectories")
print("-" * 40)
traj_req = AvailableTrajectoriesRequest(scene_id=scene_id)
traj_response = await stub.get_available_trajectories(traj_req)
if traj_response.available_trajectories:
for i, traj_info in enumerate(traj_response.available_trajectories):
poses_count = len(traj_info.trajectory.poses)
print(f" [{i}] Trajectory ID: {traj_info.trajectory_idx}")
print(f" Pose count: {poses_count}")
if poses_count > 0:
first_pose = traj_info.trajectory.poses[0]
last_pose = traj_info.trajectory.poses[-1]
print(f" Time range: {first_pose.timestamp_us}μs - {last_pose.timestamp_us}μs")
else:
print(" No available trajectories")
# 3. Get available cameras
print("\n3. Available cameras")
print("-" * 40)
cam_req = AvailableCamerasRequest(scene_id=scene_id)
cam_response = await stub.get_available_cameras(cam_req)
if cam_response.available_cameras:
for i, cam in enumerate(cam_response.available_cameras):
print(f" [{i}] Camera logical ID: {cam.logical_id}")
else:
print(" No available cameras")
# 4. Get dynamic objects
print("\n4. Dynamic objects")
print("-" * 40)
dynamic_req = AvailableDynamicObjectsRequest(scene_id=scene_id)
dynamic_response = await stub.get_dynamic_objects(dynamic_req)
if dynamic_response.dynamic_objects:
for i, obj in enumerate(dynamic_response.dynamic_objects):
print(f" [{i}] Object ID: {obj.id}")
print(f" Asset ID: {obj.asset_id}")
print(f" Semantic class: {obj.semantic_class}")
else:
print(" No dynamic objects")
# 5. Get external asset objects
print("\n5. External asset objects")
print("-" * 40)
assets_req = ExternalAssetObjectsRequest(scene_id=scene_id)
assets_response = await stub.get_external_asset_objects(assets_req)
if assets_response.track_ids:
for i, track_id in enumerate(assets_response.track_ids):
print(f" [{i}] ID: {track_id}")
else:
print(" No external asset objects")
# 6. External service information
print("\n6. External services")
print("\n" + "=" * 60)
print("Query completed")
print("=" * 60)
except Exception as e:
print(f"error: {e}")
finally:
await channel.close()
async def main():
await query_all_available_objects()
if __name__ == "__main__":
asyncio.run(main())
The USDZ generated by HuggingFace can be used to query inserted dynamic objects, but the USDZ of free data cannot be queried.
I don’t know what the problem is. I hope you can take a look. Thank you.
