Image resize and image rectify publish empty images

[eloi3]

When I launch the image resize and rectify nodes:
/right/image_raw has valid image data

/right/image_resize is published with image all 0s
/right/image_rect is published with image all 0s
(if I remove the resize node image rect is still 0s)

I am using a modified version of this launchfile: isaac_ros_dnn_stereo_depth/isaac_ros_ess/launch/isaac_ros_argus_ess.launch.py at main · NVIDIA-ISAAC-ROS/isaac_ros_dnn_stereo_depth · GitHub

import launch
from launch_ros.actions import ComposableNodeContainer
from launch_ros.descriptions import ComposableNode


def generate_launch_description():
    output_width = 480
    output_height = 288

    argus_left_node = ComposableNode(
        name='argus_mono_left',
        package='isaac_ros_argus_camera',
        plugin='nvidia::isaac_ros::argus::ArgusMonoNode',
        parameters=[{
                'mode': 4, 
                'camera_id': 1,
                'camera_info_url': 'file:///workspaces/isaac_ros-dev/left_camera.ini',
                'camera_link_frame_name': 'camera_link',
                'optical_frame_name': 'camera_left_link_optical'
        }],
        remappings=[
                    ('/left/image_raw', '/left/image_raw'),
                    ('/left/camera_info', '/left/camera_info'),
                ],
    )
    
    argus_right_node = ComposableNode(
        name='argus_mono_right',
        package='isaac_ros_argus_camera',
        plugin='nvidia::isaac_ros::argus::ArgusMonoNode',
        parameters=[{
                'mode': 4, 
                'camera_id': 0,
                'camera_info_url': 'file:///workspaces/isaac_ros-dev/right_camera.ini',
                'camera_link_frame_name': 'camera_link',
                'optical_frame_name': 'camera_right_link_optical'
        }],
        remappings=[
                    ('/left/image_raw', '/right/image_raw'),
                    ('/left/camera_info', '/right/camera_info'),
                ],
    )

    left_resize_node = ComposableNode(
        name='left_resize_node',
        package='isaac_ros_image_proc',
        plugin='nvidia::isaac_ros::image_proc::ResizeNode',
        parameters=[{
            'output_width': output_width,
            'output_height': output_height,
            'keep_aspect_ratio': True
        }],
        remappings=[
            ('camera_info', 'left/camera_info'),
            ('image', 'left/image_raw'),
            ('resize/camera_info', 'left/camera_info_resize'),
            ('resize/image', 'left/image_resize')]
    )

    right_resize_node = ComposableNode(
        name='right_resize_node',
        package='isaac_ros_image_proc',
        plugin='nvidia::isaac_ros::image_proc::ResizeNode',
        parameters=[{
            'output_width': output_width,
            'output_height': output_height,
            'keep_aspect_ratio': True
        }],
        remappings=[
            ('camera_info', 'right/camera_info'),
            ('image', 'right/image_raw'),
            ('resize/camera_info', 'right/camera_info_resize'),
            ('resize/image', 'right/image_resize')]
    )

    left_rectify_node = ComposableNode(
        name='left_rectify_node',
        package='isaac_ros_image_proc',
        plugin='nvidia::isaac_ros::image_proc::RectifyNode',
        parameters=[{
            'output_width': output_width,
            'output_height': output_height,
        }],
        remappings=[
            ('image_raw', 'left/image_resize'),
            ('camera_info', 'left/camera_info_resize'),
            ('image_rect', 'left/image_rect'),
            ('camera_info_rect', 'left/camera_info_rect')
        ]
    )

    right_rectify_node = ComposableNode(
        name='right_rectify_node',
        package='isaac_ros_image_proc',
        plugin='nvidia::isaac_ros::image_proc::RectifyNode',
        parameters=[{
            'output_width': output_width,
            'output_height': output_height,
        }],
        remappings=[
            ('image_raw', 'right/image_resize'),
            ('camera_info', 'right/camera_info_resize'),
            ('image_rect', 'right/image_rect'),
            ('camera_info_rect', 'right/camera_info_rect')
        ]
    )
    
    
    argus_mono_container = ComposableNodeContainer(
            name='argus_mono_container',
            package='rclcpp_components',
            executable='component_container_mt',
            composable_node_descriptions=[argus_left_node, argus_right_node, left_resize_node, right_resize_node, left_rectify_node, right_rectify_node],
            namespace='',
            output='screen',
            arguments=['--ros-args', '--log-level', 'info'],
        )
    return launch.LaunchDescription([argus_mono_container])

What is wrong? I attach execution logs
launch_camera_logs.txt (83.5 KB)

[Raffaello]

Hi @eloi3

Have you checked if your Argus camera is streaming?
The issue may be related to your camera and not to the Image rectify package. Check our Isaac ROS Argus camera troubleshooting guide to determine if your camera is functioning correctly.

https://nvidia-isaac-ros.github.io/repositories_and_packages/isaac_ros_argus_camera/isaac_ros_argus_camera/index.html#troubleshooting

Let me know, or please share your log to better understand the reason for these empty messages.

Best,
Raffaello

[eloi3]

Hi @Raffaello

Camera is working properly and I have streamed it using argus successfully and run other packages on the output like camera calibration. If I check:
/left/image_raw, output from argus is OK
/left/camera_info output from argus is OK (loads ini file correctly and publishes to topic)
/right/image_raw, output from argus is OK
/right/camera_info output from argus is OK (loads ini file correctly and publishes to topic)

From the resize/rectify nodes:
/left/camera_info_rect is OK
/left/camera_info_resize is OK
/right/camera_info_rect is OK
/right/camera_info_resize is OK

/left/image_resize is a black image full of 0
/left/image_rect is a black image full of 0
/right/image_resize is a black image full of 0
/right/image_rect is a black image full of 0

The log is attached in the original message.
launch_camera_logs.txt (83.5 KB)

Given left/image_raw is correct and left/camera_info too, what could be the reason for rectify node to output an image full of 0s?

Thank you

Update: After launching this exact launchfile a lot of times, some times one of the rectify nodes outputs correct output on /left/image_rect while the /right/image_rect is still outputing 0s. What could be the cause of this behavior?

[eloi3]

@Raffaello Actually right/image_resize and left/image_resize are good.

Only right/image_rect and /left/image_rect are failing

(From top to bottom: left/image_resize, left/image_rect, right/image_resize, right/image_rect)

[clyde]

I am seeing something similar.

When I set up the NITROS pipeline ArgusMonoNode => RectifyNode => AprilTagNode the output of RectifyNode often appears as black (all 0’s), but on some runs it appears fine, and on one run I saw mostly 0’s but with a weird stretched image in one region:

The simpler pipeline ArgusMonoNode => AprilTagNode works just fine.

I have a Jetson Orin Nano 8GB Dev Kit, and I’m running JetPack 6.0 and Isaac ROS.

Here is my launch file:

from launch import LaunchDescription
from launch_ros.substitutions import FindPackageShare
from launch.substitutions import PathJoinSubstitution
from launch_ros.actions import ComposableNodeContainer
from launch_ros.descriptions import ComposableNode
from launch.actions import OpaqueFunction

def launch_setup(context):
    camera_info = PathJoinSubstitution([FindPackageShare('isaac_rpi_bringup'), 'config', 'rpi_1280x720.yaml'])

    camera_node = ComposableNode(
        name='isaac_camera',
        package='isaac_ros_argus_camera',
        plugin='nvidia::isaac_ros::argus::ArgusMonoNode',
        parameters=[{
            'camera_id': 0,
            'mode': 4,  # See list above
            'camera_info_url': f"file://{camera_info.perform(context)}",
            'optical_frame_name': 'rpi_camera',
        }],
        remappings=[
            ('left/image_raw', 'image_raw'),
            ('left/camera_info', 'camera_info'),
        ],
    )

    rectify_node = ComposableNode(
        name='isaac_rectify',
        package='isaac_ros_image_proc',
        plugin='nvidia::isaac_ros::image_proc::RectifyNode',
        parameters=[{
            'output_width': 1280,
            'output_height': 720,
        }],
    )

    apriltag_node = ComposableNode(
        name='isaac_apriltag',
        package='isaac_ros_apriltag',
        plugin='nvidia::isaac_ros::apriltag::AprilTagNode',
        parameters=[{
            'size': 0.1,
            'max_tags': 64,
            'tile_size': 4,
        }],
        remappings=[
            ('image', 'image_rect'),
            ('camera_info', 'camera_info_rect'),
        ],
    )

    nitros_container = ComposableNodeContainer(
        name='nitros_container',
        package='rclcpp_components',
        executable='component_container_mt',
        composable_node_descriptions=[camera_node, rectify_node, apriltag_node],
        namespace='',
        output='screen',
        # arguments=['--ros-args', '--log-level', 'isaac_rectify:=debug']
    )

    return [nitros_container]

def generate_launch_description():
    launch_description = LaunchDescription()
    launch_description.add_action(OpaqueFunction(function=launch_setup))
    return launch_description

Here is my camera calibration file:

image_width: 1280
image_height: 720
camera_name: rpi_camera
camera_matrix:
  rows: 3
  cols: 3
  data: [786.0494 ,   0.     , 688.41104,
           0.     , 784.37525, 461.97678,
           0.     ,   0.     ,   1.     ]
distortion_model: plumb_bob
distortion_coefficients:
  rows: 1
  cols: 5
  data: [-0.315841, 0.089443, -0.000093, -0.000332, 0.000000]
rectification_matrix:
  rows: 3
  cols: 3
  data: [1., 0., 0.,
         0., 1., 0.,
         0., 0., 1.]
projection_matrix:
  rows: 3
  cols: 4
  data: [593.72913,   0.     , 709.64563,   0.     ,
           0.     , 708.18652, 477.11753,   0.     ,
           0.     ,   0.     ,   1.     ,   0.     ]

Thanks.