Hi there,
I’m trying to take a side-by-side 30fps 720p stereo video feed, run through ESS Disparity and VSLAM nodes, and then feed the outputs into NVblox to get a 3D reconstruction in realtime.
I’ve noticed that the output of the ESS Disparity node can be very noisy / intermittent. Some frames of the Disparity output are crisp, and accurate, while others have much artifacting and a “flashing” behavior. This is especially the case during motion, as shown: Notice the several frames where there is a freezing behaviour, and there are many blotches in the disparitymap.
ESS Model on 720p video stream, running disparity graph:
Using the light ESS model and 360p video input works a bit better:
Light ESS model on 360p video stream, when running the Disparity graph only:
But running Nvblox slows the pipeline down, and increases the noise seen in the disparity map . I’m trying this pipeline on a Jetson AGX Orin 64GB Dev kit on MAXN mode, but am seeing similar issues on my x86 machine as well. The cameras are correctly calibrated, rectified, and hardware synchronized.
Light ESS model on 360p video stream, when running the entire pipeline (ESS, VSLAM, Nvblox): more noise and freezing than above
Same as above, later on in the video: running Nvblox pipeline slows down disparity’ nodes output.
How can we reduce this flashing / noise issue in the Disparity map? Additionally, do you have any suggestions on how to filter this Disparity map for input to Nvblox?
My launch file:
"""Launch a Node with parameters and remappings."""
from launch import LaunchDescription
from launch.actions import ExecuteProcess
from launch_ros.actions import Node, ComposableNodeContainer
from launch_ros.descriptions import ComposableNode
NETWORK_WIDTH = 480
NETWORK_HEIGHT = 288
def generate_launch_description():
gstreamer_node = Node(
package='gscam2',
executable='gscam_main',
output='screen',
name='gscam_publisher',
parameters=[
{
"camera_info_url": "", # Camera calibration information
# Laptop pipeline
# "gscam_config": "rtpbin name=rtpbin udpsrc caps=\"application/x-rtp, media=(string)video, clock-rate=(int)90000, encoding-name=(string)H264\" port=5005 ! rtpbin.recv_rtp_sink_0 rtpbin. ! rtph264depay ! decodebin ! videoconvert ! video/x-raw, format=RGB",
# Jetson pipeline
"gscam_config": "rtpbin name=rtpbin udpsrc caps=\"application/x-rtp, media=(string)video, clock-rate=(int)90000, encoding-name=(string)H264\" port=5005 ! rtpbin.recv_rtp_sink_0 rtpbin. ! rtph264depay ! nvv4l2decoder disable-dpb=true enable-max-performance=true ! queue ! nvvidconv ! video/x-raw,format=YUY2",
"preroll": False,
"use_gst_timestamps": False,
"frame_id": "stereocamera_left_optical_frame",
"image_encoding": "yuv422_yuy2", # On Jetson
"sync_sink": False
},
],
remappings=[
('/camera_info', '/dummy/camera_info') # Don't want to use original camera_info message
]
)
splitter_node = Node(
package='custom_pkg',
executable='stereo_splitter',
parameters=[
{'left_camera_file': '/workspaces_2/isaac_ros-dev/src/custom_pkg/config/stereo_360p_left.yaml',
'right_camera_file': '/workspaces_2/isaac_ros-dev/src/custom_pkg/config/stereo_360p_right.yaml'}
]
)
rviz_node = Node(
package='rviz2',
executable='rviz2',
arguments=['-d', "/workspaces_2/isaac_ros-dev/src/custom_pkg/config/nvblox.rviz"]
)
disparity_visualizer_node = Node(
package='custom_pkg',
executable='disparity_visualizer',
)
left_rectify_node = ComposableNode(
name='left_rectify_node',
package='isaac_ros_image_proc',
plugin='nvidia::isaac_ros::image_proc::RectifyNode',
parameters=[{
'output_width': 640,
'output_height': 360,
}],
remappings=[
('image_raw', '/left/image_raw'),
('camera_info', '/left/camera_info'),
('image_rect', '/left/image_rect'),
('camera_info_rect', '/left/camera_info_rect')
]
)
right_rectify_node = ComposableNode(
name='right_rectify_node',
package='isaac_ros_image_proc',
plugin='nvidia::isaac_ros::image_proc::RectifyNode',
parameters=[{
'output_width': 640,
'output_height': 360,
}],
remappings=[
('image_raw', '/right/image_raw'),
('camera_info', '/right/camera_info'),
('image_rect', '/right/image_rect'),
('camera_info_rect', '/right/camera_info_rect')
]
)
left_rectify_image_proc_node = ComposableNode(
name='left_rectify_image_proc_node',
package='image_proc',
plugin='image_proc::RectifyNode',
remappings=[
('image', '/left/image_raw'),
('camera_info', '/left/camera_info'),
('image_rect', '/left/image_rect'),
]
)
right_rectify_image_proc_node = ComposableNode(
name='right_rectify_image_proc_node',
package='image_proc',
plugin='image_proc::RectifyNode',
remappings=[
('image', '/right/image_raw'),
('camera_info', '/right/camera_info'),
('image_rect', '/right/image_rect'),
]
)
left_image_resize_node = ComposableNode(
name='left_image_resize_node',
package='isaac_ros_image_proc',
plugin='nvidia::isaac_ros::image_proc::ResizeNode',
parameters=[{
'output_width': NETWORK_WIDTH,
'output_height': NETWORK_HEIGHT,
'keep_aspect_ratio': False # Do not add black borders to image
}],
remappings=[
('camera_info', '/left/camera_info'),
('image', '/left/image_rect'),
('resize/camera_info', '/left/camera_info_rect_resize'),
('resize/image', '/left/image_rect_resize')]
)
right_image_resize_node = ComposableNode(
name='right_image_resize_node',
package='isaac_ros_image_proc',
plugin='nvidia::isaac_ros::image_proc::ResizeNode',
parameters=[{
'output_width': NETWORK_WIDTH,
'output_height': NETWORK_HEIGHT,
'keep_aspect_ratio': False # Do not add black borders to image
}],
remappings=[
('camera_info', '/right/camera_info'),
('image', '/right/image_rect'),
('resize/camera_info', '/right/camera_info_rect_resize'),
('resize/image', '/right/image_rect_resize')]
)
disparity_node = ComposableNode(
name='disparity',
package='isaac_ros_ess',
plugin='nvidia::isaac_ros::dnn_stereo_depth::ESSDisparityNode',
parameters=[{
'engine_file_path': "/workspaces_2/isaac_ros-dev/src/isaac_ros_dnn_stereo_depth/resources/light_ess.engine",
"threshold": 0.0,
"image_type": "BGR_U8",
}],
remappings=[
('left/camera_info', '/left/camera_info_rect_resize'),
('left/image_rect', '/left/image_rect_resize'),
('right/camera_info', '/right/camera_info_rect_resize'),
('right/image_rect', '/right/image_rect_resize')
]
)
disparity_to_depth_node = ComposableNode(
name='disparity_to_depth_node',
package='isaac_ros_stereo_image_proc',
plugin='nvidia::isaac_ros::stereo_image_proc::DisparityToDepthNode',
)
left_grayscale_node = ComposableNode(
name='left_grayscale_node',
package='isaac_ros_image_proc',
plugin='nvidia::isaac_ros::image_proc::ImageFormatConverterNode',
parameters=[{
'encoding_desired': "mono8",
}],
remappings=[
('image_raw', '/left/image_raw'),
('camera_info', '/left/camera_info'),
('image', '/left/image_raw_grayscale'),
]
)
right_grayscale_node = ComposableNode(
name='right_grayscale_node',
package='isaac_ros_image_proc',
plugin='nvidia::isaac_ros::image_proc::ImageFormatConverterNode',
parameters=[{
'encoding_desired': "mono8",
}],
remappings=[
('image_raw', '/right/image_raw'),
('camera_info', '/right/camera_info'),
('image', '/right/image_raw_grayscale'),
]
)
visual_slam_node = ComposableNode(
name='visual_slam_node',
package='isaac_ros_visual_slam',
plugin='nvidia::isaac_ros::visual_slam::VisualSlamNode',
remappings=[('stereo_camera/left/image', '/left/image_raw_grayscale'), # need grayscale images
('stereo_camera/left/camera_info', '/left/camera_info'),
('stereo_camera/right/image', '/right/image_raw_grayscale'),
('stereo_camera/right/camera_info', '/right/camera_info')],
parameters=[{
'enable_rectified_pose': True,
'denoise_input_images': False,
'rectified_images': False,
'img_jitter_threshold_ms': 110.00,
'enable_slam_visualization': True,
'enable_observations_view': True,
'enable_landmarks_view': True,
"enable_localization_n_mapping": True,
'map_frame': 'map',
'odom_frame': 'odom',
'base_frame': 'base_link', # robot center
"input_left_camera_frame": "stereocamera_left_frame", # Not the optical frame
}],
)
# Nvblox node
nvblox_node = ComposableNode(
name='nvblox_node',
package='nvblox_ros',
plugin='nvblox::NvbloxNode',
parameters = ["/workspaces_2/isaac_ros-dev/src/custom_pkg/config/nvblox_params.yaml"],
remappings=[
('color/image', '/left/image_rect_resize'),
('color/camera_info', '/left/camera_info_rect_resize'),
('depth/camera_info', '/left/camera_info_rect_resize'),
('depth/image', '/depth'),
]
)
nitros_container = ComposableNodeContainer(
name='nitros_container',
package='rclcpp_components',
namespace='',
executable='component_container_mt',
composable_node_descriptions=[
# left_rectify_node, right_rectify_node, # Isaac ROS version
left_rectify_image_proc_node, right_rectify_image_proc_node, # Original ROS image_proc version
left_image_resize_node, right_image_resize_node,
disparity_node,
disparity_to_depth_node,
left_grayscale_node, right_grayscale_node,
visual_slam_node,
nvblox_node
],
output='screen'
)
return LaunchDescription([
gstreamer_node,
splitter_node,
nitros_container,
rviz_node,
disparity_visualizer_node,
])
