ROS - How to cross compile ROS for DRIVE AGX Developer Kit with DRIVE Software 10.0

Preparing a Cross-Compilation Sysroot

  • installing arm64 Debian dependencies with qemu-chroot
    export SYSROOT=~/nvidia/nvidia_sdk/DRIVE_Software_10.0_Linux_OS_DDPX/DRIVEOS/drive-t186ref-linux/targetfs_a
    cd $SYSROOT
     
    sudo apt-get install qemu-user-static
    sudo cp /usr/bin/qemu-aarch64-static usr/bin/
    sudo cp -ib /etc/resolv.conf etc
    sudo mount -o bind /dev dev
    sudo mount -o bind /proc proc
    sudo mount -o bind /sys sys
    
    sudo LC_ALL=C chroot .
    # apt-get update
    # apt-get install libboost-all-dev libtinyxml-dev libtinyxml2-dev liblz4-dev libbz2-dev libapr1 libaprutil1 libconsole-bridge-dev libpoco-dev libgpgme-dev python-defusedxml python-rospkg python-catkin-pkg python-netifaces liblog4cxx-dev
    # exit
    
    sudo umount sys proc dev 
    sudo rm usr/bin/qemu-aarch64-static
    sudo mv etc/resolv.conf~ etc/resolv.conf
    sudo rm -rf var/lib/apt/lists/*
    sudo rm -rf dev/*
    sudo rm -rf var/log/*
    sudo rm -rf var/tmp/*
    sudo rm -rf var/cache/apt/archives/*.deb
    sudo rm -rf tmp/*
    

Fixing Broken Symlinks

  • When dependencies are installed with qemu-chroot, the root file system may contain symlinks similar to the following: /usr/lib/aarch64-linux-gnu/libXXXX.so -> /lib/aarch64-linux-gnu/libXXXX.so.versio.no. These symlinks resolve on the arm64 platform but are broken on the host system. Attempts to link libXXXX.so are unsuccessful.To correct broken symlinks
  • The broken symlinks can be fixed temporarily with overlays, using commands similar to the following:
    sudo mkdir /lib/aarch64-linux-gnu
    sudo mkdir /tmp/ros-cc-overlayfs
    sudo mount -t overlay -o lowerdir=$SYSROOT/lib/aarch64-linux-gnu,upperdir=/lib/aarch64-linux-gnu,workdir=/tmp/ros-cc-overlayfs overlay /lib/aarch64-linux-gnu
    

Downloading Sources

  • ROS Bare Bone source code can be downloaded from the upstream repositories using the python utilities python-rosinstall-generator and python-wstool.
  • To install python-rosinstall-generator and python-wstool
    sudo sh -c 'echo \
    "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list'
    sudo apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654
    sudo apt-get update
    
    sudo apt install python-rosdep python-rosinstall-generator python-wstool python-rosinstall build-essential python-empy libgtest-dev
    
  • To download the source code
    mkdir $HOME/ros_catkin_ws && cd $HOME/ros_catkin_ws
    rosinstall_generator ros_comm --rosdistro melodic --deps --tar > melodic-ros_comm.rosinstall
    wstool init -j8 src melodic-ros_comm.rosinstall
    

Invoking the Cross Compiler

  • create Toolchain-V5L.cmake with below content for cross compilation
    set(CMAKE_SYSTEM_NAME Linux)
    # Specify the cross compiler 
    set(TOOLCHAIN "$ENV{HOME}/nvidia/nvidia_sdk/DRIVE_Software_10.0_Linux_OS_E3550/DRIVEOS/toolchains/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu")                                              
    set(CMAKE_CXX_COMPILER "${TOOLCHAIN}/bin/aarch64-linux-gnu-g++")
    set(CMAKE_C_COMPILER "${TOOLCHAIN}/bin/aarch64-linux-gnu-gcc")
    # Targetfs path 
    set(ROS_SYSROOT "$ENV{HOME}/nvidia/nvidia_sdk/DRIVE_Software_10.0_Linux_OS_E3550/DRIVEOS/drive-t186ref-linux/targetfs_a")
    # Library paths 
    set(LD_PATH "${ROS_SYSROOT}/usr/lib/aarch64-linux-gnu")
    set(LD_PATH_EXTRA "${ROS_SYSROOT}/lib/aarch64-linux-gnu")
    # setup compiler for cross-compilation 
    set(CMAKE_CXX_FLAGS           "-fPIC"               CACHE STRING "c++ flags")
    set(CMAKE_C_FLAGS             "-fPIC"               CACHE STRING "c flags")
    set(CMAKE_SHARED_LINKER_FLAGS ""                    CACHE STRING "shared linker flags")
    set(CMAKE_MODULE_LINKER_FLAGS ""                    CACHE STRING "module linker flags")
    set(CMAKE_EXE_LINKER_FLAGS    ""                    CACHE STRING "executable linker flags")
    set(CMAKE_FIND_ROOT_PATH ${ROS_SYSROOT})
    # Set compiler flags 
    set(CMAKE_SHARED_LINKER_FLAGS   "--sysroot=${CMAKE_FIND_ROOT_PATH} -L${LD_PATH} -L${LD_PATH_EXTRA} -Wl,-rpath,${LD_PATH} -Wl,-rpath,${LD_PATH_EXTRA} -Wl,-rpath,${LD_PATH} -Wl,-rpath,${LD_PATH_EXTRA} ${CMAKE_SHARED_LINKER_FLAGS}")
    set(CMAKE_MODULE_LINKER_FLAGS   "--sysroot=${CMAKE_FIND_ROOT_PATH} -L${LD_PATH} -L${LD_PATH_EXTRA} -Wl,-rpath,${LD_PATH} -Wl,-rpath,${LD_PATH_EXTRA} -Wl,-rpath,${LD_PATH} -Wl,-rpath,${LD_PATH_EXTRA} ${CMAKE_SHARED_LINKER_FLAGS}")
    set(CMAKE_EXE_LINKER_FLAGS      "--sysroot=${CMAKE_FIND_ROOT_PATH} -L${LD_PATH} -L${LD_PATH_EXTRA} -Wl,-rpath,${LD_PATH} -Wl,-rpath,${LD_PATH_EXTRA} -Wl,-rpath,${LD_PATH_EXTRA} ${CMAKE_EXE_LINKER_FLAGS}")
    
    set(CMAKE_C_FLAGS "-fPIC --sysroot=${CMAKE_FIND_ROOT_PATH} -fpermissive -g" CACHE INTERNAL "" FORCE)
    set(CMAKE_CXX_FLAGS "-fPIC --sysroot=${CMAKE_FIND_ROOT_PATH} -fpermissive -g" CACHE INTERNAL "" FORCE)
    # Search for programs only in the build host directories 
    set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
    # Search for libraries and headers only in the target directories 
    set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
    set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
    
  • To cross-compile ROS
    ./src/catkin/bin/catkin_make_isolated --install -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=<Toolchain-V5L.cmake file path>
    
  • After cross-compilation is complete, revert the overlayfs mount with the following commands:
    sudo umount /lib/aarch64-linux-gnu
    sudo rm -rf /tmp/ros-cc-overlayfs
    

Verifying

  • scp ~/ros_catkin_ws/install_isolated directory to the target and refer to http://wiki.ros.org/ROS/Tutorials to further verification.

Dear VickNV,

Thank you for your kind guidance, it’s very helpful for me. After the step of “Preparing a Cross-Compilation Sysroot” and “Fixing Broken Symlinks”, I tried to install ROS Melodic using the following command sudo apt install ros-melodic-desktop instead of using python, folloing error of unmet dependencies was encountered. BTW, my host PC has installed DRIVE Software 10.0 via SDK Manager.

sophia@ZBook:~/nvidia/nvidia_sdk/DRIVE_Software_10.0_Linux_OS_E3550/DRIVEOS/drive-t186ref-linux/targetfs_a$ sudo apt install ros-melodic-desktop
Reading package lists... Done
Building dependency tree       
Reading state information... Done
Some packages could not be installed. This may mean that you have
requested an impossible situation or if you are using the unstable
distribution that some required packages have not yet been created
or been moved out of Incoming.
The following information may help to resolve the situation:

The following packages have unmet dependencies:
 ros-melodic-desktop : Depends: ros-melodic-common-tutorials but it is not going to be installed
                       Depends: ros-melodic-geometry-tutorials but it is not going to be installed
                       Depends: ros-melodic-robot but it is not going to be installed
                       Depends: ros-melodic-ros-tutorials but it is not going to be installed
                       Depends: ros-melodic-urdf-tutorial but it is not going to be installed
                       Depends: ros-melodic-visualization-tutorials but it is not going to be installed
                       Depends: ros-melodic-viz but it is not going to be installed

As for the error, I have tried several methods to solve it,like update package, all didn’t work. But when I tried to install ROS Melodic in emulated Root command line, no errors similar with upper unmet dependencies occur. I feel confused and want to figure it out what the difference is. And if there’s better method to solve the unmet dependencies problem.

Hi pan.zhang,

Is this issue caused by the cross compilation steps here or just a ROS installation issue?

If it’s a ros installation issue, could you search ROS community sites to see if any discussion is related to this issue? Thanks!

I have the same issue as pan.zhang – I can’t install ROS melodic or any of its dependencies while in the chroot.

Also, the /etc/apt/sources.list inside targetfs_a is empty, so I’m not sure how apt-get would be able to find any dependencies anyway.

I don’t think this is a ROS installation issue because of the empty sources.list, and because I could install ROS melodic just fine on my host machine.

Do you have any updates on this process?

Hi evanmt03s,

Why do you need to install ROS melodic in chroot? This topic is about source installation of cross compilation. You can scp them to the target after cross compiling. Thanks!

I may have been trying to do something that wasn’t required.

But this command (run in chroot):

apt-get install libboost-all-dev libtinyxml-dev libtinyxml2-dev liblz4-dev libbz2-dev libapr1 libaprutil1 libconsole-bridge-dev libpoco-dev libgpgme-dev python-defusedxml python-rospkg python-catkin-pkg python-netifaces liblog4cxx-dev

also fails. apt-get can’t find these packages.

How should I proceed?

Thanks!

The file isn’t empty on my side. Could you help to check?

~/nvidia/nvidia_sdk/DRIVE_Software_10.0_Linux_OS_E3550/DRIVEOS/drive-t186ref-linux/targetfs_a/etc/apt/sources.list

These are the contents of that file now – I was just finishing a fresh install of the DriveWorks target tools after previous failures, which must be why it’s no longer empty:

# deb cdrom:[Ubuntu 18.04.4 LTS _Bionic Beaver_ - Release amd64 (20200203.1)]/ bionic main restricted

# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
# newer versions of the distribution.
deb [arch=amd64,i386] http://us.archive.ubuntu.com/ubuntu/ bionic main restricted
# deb-src http://us.archive.ubuntu.com/ubuntu/ bionic main restricted

## Major bug fix updates produced after the final release of the
## distribution.
deb [arch=amd64,i386] http://us.archive.ubuntu.com/ubuntu/ bionic-updates main restricted
# deb-src http://us.archive.ubuntu.com/ubuntu/ bionic-updates main restricted

## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
## team. Also, please note that software in universe WILL NOT receive any
## review or updates from the Ubuntu security team.
deb [arch=amd64,i386] http://us.archive.ubuntu.com/ubuntu/ bionic universe
# deb-src http://us.archive.ubuntu.com/ubuntu/ bionic universe
deb [arch=amd64,i386] http://us.archive.ubuntu.com/ubuntu/ bionic-updates universe
# deb-src http://us.archive.ubuntu.com/ubuntu/ bionic-updates universe

## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu 
## team, and may not be under a free licence. Please satisfy yourself as to 
## your rights to use the software. Also, please note that software in 
## multiverse WILL NOT receive any review or updates from the Ubuntu
## security team.
deb [arch=amd64,i386] http://us.archive.ubuntu.com/ubuntu/ bionic multiverse
# deb-src http://us.archive.ubuntu.com/ubuntu/ bionic multiverse
deb [arch=amd64,i386] http://us.archive.ubuntu.com/ubuntu/ bionic-updates multiverse
# deb-src http://us.archive.ubuntu.com/ubuntu/ bionic-updates multiverse

## N.B. software from this repository may not have been tested as
## extensively as that contained in the main release, although it includes
## newer versions of some applications which may provide useful features.
## Also, please note that software in backports WILL NOT receive any review
## or updates from the Ubuntu security team.
deb [arch=amd64,i386] http://us.archive.ubuntu.com/ubuntu/ bionic-backports main restricted universe multiverse
# deb-src http://us.archive.ubuntu.com/ubuntu/ bionic-backports main restricted universe multiverse

## Uncomment the following two lines to add software from Canonical's
## 'partner' repository.
## This software is not part of Ubuntu, but is offered by Canonical and the
## respective vendors as a service to Ubuntu users.
# deb http://archive.canonical.com/ubuntu bionic partner
# deb-src http://archive.canonical.com/ubuntu bionic partner

deb [arch=amd64,i386] http://security.ubuntu.com/ubuntu bionic-security main restricted
# deb-src http://security.ubuntu.com/ubuntu bionic-security main restricted
deb [arch=amd64,i386] http://security.ubuntu.com/ubuntu bionic-security universe
# deb-src http://security.ubuntu.com/ubuntu bionic-security universe
deb [arch=amd64,i386] http://security.ubuntu.com/ubuntu bionic-security multiverse
# deb-src http://security.ubuntu.com/ubuntu bionic-security multiverse
deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable
# deb-src [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable

Does this look correct? Thanks for your help!

If anyone encounters the apt issue of DRIVE Software 10.0, please follow below post to work around it. Thanks!

@VickNV

On my fresh SDK Drive AGX install. I notice I dont have the DRIVE_Software_10.0_Linux_OS_E3550 but I do have the DRIVE_Software_10.0_Linux_OS_DDPX folder which matches the same contents.

Going further down on the procedure. I am getting the following messages:

root@obelisk:/usr/lib/aarch64-linux-gnu# sudo mkdir /lib/aarch64-linux-gnu
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown QEMU_IFLA_BR type 46
Unknown QEMU_IFLA_BR type 41
Unknown QEMU_IFLA_BR type 45
Unknown QEMU_IFLA_BR type 42
Unknown QEMU_IFLA_BR type 43
Unknown QEMU_IFLA_BR type 44
/bin/mkdir: cannot create directory ‘/lib/aarch64-linux-gnu’: File exists
root@obelisk:/usr/lib/aarch64-linux-gnu# sudo mkdir /tmp/ros-cc-overlayfs
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown QEMU_IFLA_BR type 46
Unknown QEMU_IFLA_BR type 41
Unknown QEMU_IFLA_BR type 45
Unknown QEMU_IFLA_BR type 42
Unknown QEMU_IFLA_BR type 43
Unknown QEMU_IFLA_BR type 44

so for fixing the symlinks: I get the following error:

root@obelisk:/tmp/ros-cc-overlayfs# sudo mount -t overlay -o lowerdir=$SYSROOT/lib/aarch64-linux-gnu,upperdir=/lib/aarch64-linux-gnu,workdir=/tmp/ros-cc-overlayfs overlay /lib/aarch64-linux-gnu
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown host QEMU_IFLA type: 50
Unknown host QEMU_IFLA type: 51
Unknown host QEMU_IFLA type: 47
Unknown host QEMU_IFLA type: 48
Unknown host QEMU_IFLA type: 43
Unknown QEMU_IFLA_BR type 46
Unknown QEMU_IFLA_BR type 41
Unknown QEMU_IFLA_BR type 45
Unknown QEMU_IFLA_BR type 42
Unknown QEMU_IFLA_BR type 43
Unknown QEMU_IFLA_BR type 44
mount: /lib/aarch64-linux-gnu: mount(2) system call failed: Too many levels of symbolic links.

Can you advise if the folder is correct and what could be causing the error on QEMU?

I haven’t seen the messages before. Is it due to “root” running with “sudo”?

@VickNV

I tried without sudo and got same output:

root@obelisk:/# mkdir /lib/aarch64-linux-gnu
/bin/mkdir: cannot create directory ‘/lib/aarch64-linux-gnu’: File exists
root@obelisk:/# mkdir /tmp/ros-cc-overlayfs
/bin/mkdir: cannot create directory ‘/tmp/ros-cc-overlayfs’: File exists
root@obelisk:/# mount -t overlay -o lowerdir=$SYSROOT/lib/aarch64-linux-gnu,upperdir=/lib/aarch64-linux-gnu,workdir=/tmp/ros-cc-overlayfs overlay /lib/aarch64-linux-gnu
mount: /lib/aarch64-linux-gnu: mount(2) system call failed: Too many levels of symbolic links.

Following with ROS steps: got also another error:

root@obelisk:/# apt --fix-broken install
Reading package lists… Done
Building dependency tree
Reading state information… Done
Correcting dependencies… Done
The following packages were automatically installed and are no longer required:
cuda-command-line-tools-10-2 cuda-compiler-10-2 cuda-cudart-dev-10-2
cuda-cuobjdump-10-2 cuda-cupti-10-2 cuda-driver-dev-10-2 cuda-gdb-10-2
cuda-gpu-library-advisor-10-2 cuda-memcheck-10-2 cuda-misc-headers-10-2
cuda-nvcc-10-2 cuda-nvdisasm-10-2 cuda-nvprof-10-2 cuda-nvprune-10-2
cuda-nvrtc-dev-10-2 cuda-nvtx-10-2 libcublas-dev libcublas10
Use ‘sudo apt autoremove’ to remove them.
The following additional packages will be installed:
python-catkin-pkg-modules python-rospkg-modules
The following NEW packages will be installed:
python-catkin-pkg-modules python-rospkg-modules
0 upgraded, 2 newly installed, 0 to remove and 302 not upgraded.
44 not fully installed or removed.
Need to get 0 B/65.9 kB of archives.
After this operation, 390 kB of additional disk space will be used.
Do you want to continue? [Y/n] Y
debconf: delaying package configuration, since apt-utils is not installed
E: Can not write log (Is /dev/pts mounted?) - posix_openpt (19: No such device)
(Reading database … 93408 files and directories currently installed.)
Preparing to unpack …/python-catkin-pkg-modules_0.4.22-1_all.deb …
Unpacking python-catkin-pkg-modules (0.4.22-1) …
dpkg: error processing archive /var/cache/apt/archives/python-catkin-pkg-modules_0.4.22-1_all.deb (–unpack):
trying to overwrite ‘/usr/lib/python2.7/dist-packages/catkin_pkg/init.py’, which is also in package python-catkin-pkg 0.3.9-1
Preparing to unpack …/python-rospkg-modules_1.2.8-1_all.deb …
Unpacking python-rospkg-modules (1.2.8-1) …
dpkg: error processing archive /var/cache/apt/archives/python-rospkg-modules_1.2.8-1_all.deb (–unpack):
trying to overwrite ‘/usr/lib/python2.7/dist-packages/rospkg/init.py’, which is also in package python-rospkg 1.1.4-1
Errors were encountered while processing:
/var/cache/apt/archives/python-catkin-pkg-modules_0.4.22-1_all.deb
/var/cache/apt/archives/python-rospkg-modules_1.2.8-1_all.deb
E: Sub-process /usr/bin/dpkg returned an error code (1)

Not sure if it is related to QEMU or mounting procedure. Can you advise?

Without “sudo”, you don’t see messages like “Unknown host QEMU_IFLA type: 50” anymore, right?

I haven’t seen the issue before. Please try with run all the steps in my initial post with non root user (using sudo as my command lines) and “export SYSROOT=~/nvidia/nvidia_sdk/DRIVE_Software_10.0_Linux_OS_DDPX/DRIVEOS/drive-t186ref-linux/targetfs_a” (matching your path after DRIVE Software 10.0 installation).

Correct. Without sudo I don’t see the QEMU_XXX messages anymore. I am going to try again the steps in couple systems where the SDK is installed and provide feedback.

@VickNV

I tried in a different system and got the same results. Can you advise?

Attaching log file with the stepsnvidia_ros.log (99.6 KB)

Thanks for sharing more complete log.

Only the three command lines need to be run in chroot. It seems you missed “exit”.

@VickNV

That’s correct. I missed the exit command. I also modified the cross compiler CMAKE file to get the correct path to the toolchain.