ROS2 Foxy Cross Compilation on AGX Orin Drivekit

Autonomous Vehicles DRIVE AGX Orin DRIVE AGX Orin General
1

DRIVE OS Version: 6.0.10

Issue Description: Hello all,

I am using the Nvidia docker container on an X86 machine to cross compile to my AGX Orin Drivekit.

I can successfully cross-compile and deploy my DW code with no problem.

I am writing a simple middle layer to communicate betwen ROS2 foxy and the DW abstraction layer. I was doing that via a DW code and a second ROS2 node that communicated through sockets, but the latency and resource consumption is too heavy. Also compiling the ROS2 code straight on the drivekit is not recommended and takes a lot longer.

I am looking for some guidance on how to cross-compile with both driveworks and ros2 libraries in the same project, namely in compiling and installing the required files in my X86 machine under the ARM targetfs.

I found some posts about it but they were all ROS 1 and over 4 years old.

Thank you very much!

2

Update for anyone trying this out:

You can use qemu to install ROS2 foxy in the targetfs and compile with it.

Here is how I did it:

export SYSROOT=/drive/drive-linux/filesystem/targetfs
mv ${SYSROOT}/etc/resolv.conf ${SYSROOT}/etc/resolv.conf.bk
sudo cp /etc/resolv.conf ${SYSROOT}/etc/resolv.conf

sudo mount -o bind /dev  ${SYSROOT}/dev
sudo mount -o bind /proc ${SYSROOT}/proc
sudo mount -o bind /sys  ${SYSROOT}/sys

sudo LC_ALL=C chroot ${SYSROOT} /bin/bash << 'EOF'

I then created a script to install ROS2 foxy based on the original ROS documentation at $SYSROOT/tmp/install_ros2.sh :

#!/bin/bash

apt install locales
locale-gen en_US en_US.UTF-8
update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
export LANG=en_US.UTF-8

apt install software-properties-common
add-apt-repository universe
apt update && apt install curl -y
curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg

echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(. /etc/os-release && echo $UBUNTU_CODENAME) main" | tee /etc/apt/sources.list.d/ros2.list > /dev/null


apt update
apt upgrade
apt install ros-foxy-ros-base python3-argcomplete



apt install ros-dev-tools
echo "ROS2 Foxy install DONE"

I then executed it in the chroot

sudo chmod +x ${SYSROOT}/tmp/install_ros2.sh
sudo LC_ALL=C chroot ${SYSROOT} /bin/bash /tmp/install_ros2.sh

And fixed the symlinks

find ${SYSROOT}/opt/ros/foxy/lib -xtype l | head -20
find ${SYSROOT} -xtype l | while read link; do     target=$(readlink "$link")
if [[ "$target" == /* ]] && [ ! -e "$target" ]; then 
    new_target="${SYSROOT}${target}";
    if [ -e "$new_target" ]; then 
        sudo ln -sfn "$new_target" "$link"; echo "Fixed: $link -> $new_target";
    fi;
fi;
done

New cmake toolchain specifically for the build

# /usr/local/driveworks/samples/cmake/Toolchain-V5L-ROS2Foxy.cmake

include(/usr/local/driveworks/samples/cmake/Toolchain-V5L.cmake)

set(TOOLCHAIN_ROOT /drive/toolchains/aarch64--glibc--stable-2022.03-1)
set(ROS_SYSROOT    ${VIBRANTE_PDK}/filesystem/targetfs)
set(ROS2_PREFIX    ${ROS_SYSROOT}/opt/ros/foxy)

# This is what --print-sysroot returns for the buildroot toolchain
set(TOOLCHAIN_SYSROOT ${TOOLCHAIN_ROOT}/aarch64-buildroot-linux-gnu/sysroot)
# The toolchain sysroot has crt*.o; targetfs has ROS2/PDK libs
set(CMAKE_C_FLAGS
    "-fPIC --sysroot=${TOOLCHAIN_SYSROOT}"
    CACHE INTERNAL "" FORCE)
set(CMAKE_CXX_FLAGS
    "-fPIC --sysroot=${TOOLCHAIN_SYSROOT}"
    CACHE INTERNAL "" FORCE)

set(CMAKE_EXE_LINKER_FLAGS
    "${CMAKE_EXE_LINKER_FLAGS} \
     --sysroot=${TOOLCHAIN_SYSROOT} \
     -L${TOOLCHAIN_SYSROOT}/usr/lib \
     -L${TOOLCHAIN_SYSROOT}/lib \
     -L${ROS2_PREFIX}/lib \
     -L${ROS_SYSROOT}/usr/lib/aarch64-linux-gnu \
     -L${ROS_SYSROOT}/lib/aarch64-linux-gnu \
     -L${VIBRANTE_PDK}/lib-target \
     -Wl,-rpath-link,${TOOLCHAIN_SYSROOT}/usr/lib \
     -Wl,-rpath-link,${TOOLCHAIN_SYSROOT}/lib \
     -Wl,-rpath-link,${ROS2_PREFIX}/lib \
     -Wl,-rpath-link,${ROS_SYSROOT}/usr/lib/aarch64-linux-gnu \
     -Wl,-rpath-link,${ROS_SYSROOT}/lib/aarch64-linux-gnu \
     -Wl,-rpath-link,${VIBRANTE_PDK}/lib-target"
    CACHE INTERNAL "" FORCE)
set(CMAKE_SHARED_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS} CACHE INTERNAL "" FORCE)
set(CMAKE_MODULE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS} CACHE INTERNAL "" FORCE)

set(CMAKE_FIND_ROOT_PATH
    ${TOOLCHAIN_SYSROOT}
    ${ROS_SYSROOT}
    ${ROS2_PREFIX}
    ${VIBRANTE_PDK}
)
list(APPEND CMAKE_PREFIX_PATH ${ROS2_PREFIX})

set(CMAKE_INSTALL_RPATH
    "/opt/ros/foxy/lib"
    "/usr/lib/aarch64-linux-gnu"
    "/usr/local/driveworks/lib"
)
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)


# In Toolchain-V5L-ROS2Foxy.cmake - explicit RMW cmake paths

# Force ament to find RMW plugins from sysroot
set(rmw_fastrtps_cpp_DIR
    ${ROS_SYSROOT}/opt/ros/foxy/share/rmw_fastrtps_cpp/cmake
    CACHE PATH "" FORCE)
set(fastrtps_DIR
    ${ROS_SYSROOT}/opt/ros/foxy/share/fastrtps/cmake
    CACHE PATH "" FORCE)
set(rmw_DIR
    ${ROS_SYSROOT}/opt/ros/foxy/share/rmw/cmake
    CACHE PATH "" FORCE)
set(rmw_implementation_DIR
    ${ROS_SYSROOT}/opt/ros/foxy/share/rmw_implementation/cmake
    CACHE PATH "" FORCE)

# Tell rmw_implementation where to find available middlewares
set(rmw_implementation_FOUND_MIDDLEWARES "rmw_fastrtps_cpp" CACHE STRING "" FORCE)

# Also set ament_index search path so it finds installed RMW resources
set(AMENT_PREFIX_PATH
    ${ROS_SYSROOT}/opt/ros/foxy
    CACHE STRING "" FORCE)

set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH)

set(ROS2_INCLUDE_DIRS
    ${ROS_SYSROOT}/opt/ros/foxy/include
    ${ROS_SYSROOT}/usr/include
    ${ROS_SYSROOT}/usr/include/aarch64-linux-gnu
)
include_directories(BEFORE SYSTEM ${ROS2_INCLUDE_DIRS})

And changed the CMakeLists.txt on the sample app:

project(proj_name)

set(SOURCES main.cpp)

set(LIBRARIES
    samples_framework
    samples_framework_checks
    samples_framework_program_arguments
    ${Driveworks_LIBRARIES}
)

find_package(rclcpp REQUIRED)
find_package(sensor_msgs REQUIRED)
find_package(rcl REQUIRED)
find_package(rcutils REQUIRED)
find_package(rosidl_typesupport_cpp REQUIRED)

add_executable(${PROJECT_NAME} ${SOURCES})

target_include_directories(${PROJECT_NAME} PRIVATE
    ${rclcpp_INCLUDE_DIRS}
    ${sensor_msgs_INCLUDE_DIRS}
    ${rcl_INCLUDE_DIRS}
    ${rcutils_INCLUDE_DIRS}
)

target_link_libraries(${PROJECT_NAME} PRIVATE
    ${LIBRARIES}

    # rclcpp stack
    ${ROS2_PREFIX}/lib/librclcpp.so
    ${ROS2_PREFIX}/lib/librcl.so
    ${ROS2_PREFIX}/lib/librcutils.so

    # rmw
    ${ROS2_PREFIX}/lib/librmw_implementation.so
    ${ROS2_PREFIX}/lib/librmw.so

    # rosidl
    ${ROS2_PREFIX}/lib/librosidl_typesupport_cpp.so
    ${ROS2_PREFIX}/lib/librosidl_typesupport_c.so
    ${ROS2_PREFIX}/lib/librosidl_runtime_c.so
    ${ROS2_PREFIX}/lib/librosidl_typesupport_fastrtps_cpp.so
    ${ROS2_PREFIX}/lib/librosidl_typesupport_fastrtps_c.so
    ${ROS2_PREFIX}/lib/librosidl_typesupport_introspection_cpp.so
    ${ROS2_PREFIX}/lib/librosidl_typesupport_introspection_c.so

    # sensor_msgs
    ${ROS2_PREFIX}/lib/libsensor_msgs__rosidl_typesupport_cpp.so
    ${ROS2_PREFIX}/lib/libsensor_msgs__rosidl_typesupport_c.so
    ${ROS2_PREFIX}/lib/libsensor_msgs__rosidl_generator_c.so
    ${ROS2_PREFIX}/lib/libsensor_msgs__rosidl_typesupport_fastrtps_cpp.so
    ${ROS2_PREFIX}/lib/libsensor_msgs__rosidl_typesupport_fastrtps_c.so
)

sdk_add_sample(${PROJECT_NAME})

My final build script is:

#!/bin/bash
set -e

rm -r /root/project_name/build/*

unset AMENT_PREFIX_PATH
unset ROS_DISTRO
unset CMAKE_PREFIX_PATH

export PYTHONPATH=/drive/drive-linux/filesystem/targetfs/opt/ros/foxy/lib/python3.8/site-packages:${PYTHONPATH}
export AMENT_PREFIX_PATH=/drive/drive-linux/filesystem/targetfs/opt/ros/foxy

cmake -B /root/project_name/build \
    -DCMAKE_TOOLCHAIN_FILE=/usr/local/driveworks/samples/cmake/Toolchain-V5L-ROS2Foxy.cmake \
    -DCMAKE_BUILD_TYPE=Release \
    -S /root/project_name/

cmake --build /root/project_name/build -- -j8

Anyone from nvidia can confirm if this is a plausible solution or possible problems with this?

Thanks

4

this is impressive, if a ROS 2 build is possible, I would imagine repacking the base image with U22.04 would also be possible as well? I have a thread asking for confirmation, unfortunately all they say is DRIVE OS 6.X.X doesn’t support so and so.

Would you mind confirming if you have been successful in applying your build?

5

I understand that you want to cross compile ros2 app nodes(rather than ros2 itself from src) with driveos toolchain called v5l toolchain so that apps will use rclcpp and driveos and runs on the drive orin device.

@p.ramos your approach is too complicated, please check this repo out meghdeepj/nv_driveworks_demo: Nvidia Driveworks Demo with CGF, ROS2 and Docker.

That’s what the repo do

  1. add precompiled ros2 foxy from foxy · Releases · ros2/ros2 to /drive/drive-linux/filesystem/targetfs/ so that find_package would find rclcpp and ament_cmake, because ros2 foxy is officially precompiled with ubuntu2004, so the binary is compatiable with the driveos sdk. if you want the higher version like humble, copy the precompiled ros2 humble from nvidia isaac-ros(which is based on ubuntu20.04) Isaac ROS Dev Base | NVIDIA NGC
  2. install arm64 version of deps, apt install -y libspdlog-dev:arm64 libyaml-dev:arm64 libpython3-dev:arm64 libtinyxml2-dev:arm64 libssl-dev:arm64 openssl:arm64 libboost-dev:arm64 libboost-thread-dev:arm64 libtclap-dev:arm64 libgoogle-glog-dev:arm64 libfl-dev:arm64 bison:arm64 libfmt-dev:arm64
  3. modity the toolchain.cmake to setup ros2 cross-compile cmake config.

then, you can compile your ros2 nodes in cmakelists like this

#-------------------------------------------------------------------------------
# ros2 packages
#-------------------------------------------------------------------------------
find_package(ament_cmake REQUIRED)
find_package(rclcpp REQUIRED)
find_package(std_msgs REQUIRED)

add_library(${PROJECT_NAME} SHARED ${DWCGF_NODES_FILES} ${DWCGF_CHANNEL_FILES})
ament_target_dependencies(${PROJECT_NAME} "rclcpp" "std_msgs" )

ament_package()
7

I might be missing something, but that approach looks pretty similar to mine, at least in steps 1 and 3.

Mostly I wanted to avoid building a full project from scratch, I have a very tight time window for the software component of my implementation and I wanted to be able to slightly modify the nvidia sample applications to directly send messages to ROS2. This approach of course also works with other libraries, not just ROS.

The original problem was that I could not compile DW on the target itself, and I could not compile ROS on my cross-compilation container in my server, so I needed seperate scripts with basic UDP socket communication between them. But the memmory and CPU usage was through the roof!

Adapting the nvidia samples to send straight to ROS allows me to connect 8x 8mp 4k30fps cameras and stream them to ROS with minimal RAM and GPU usage and less than 12% of a CPU core being used! I also did the same for my RADARs and now my LiDAR and got similarly great performance.