Cross compiling for Xavier NX (recompile with -fPIC, linker error)

Autonomous Machines Jetson & Embedded Systems Jetson Xavier NX
1

Hello,

I am trying to set-up cross compile environment, so I can pre-build various libraries for xavier nx (target system) on more powerful host system - Ubuntu 18.04 x86_64. So I can create my own system image with pre-installed libraries which I need etc.

I am testing it with ArUco library, which is not heavily dependent on other libraries and is quite small project.

I added toolchain.cmake file into cmake dir inside aruco folder:

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR aarch64)

set(CMAKE_SYSROOT /home/edo/rootfs-xavier)

set(tools /home/edo/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu)
set(CMAKE_C_COMPILER ${tools}/bin/aarch64-linux-gnu-gcc)
set(CMAKE_CXX_COMPILER ${tools}/bin/aarch64-linux-gnu-g++)

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

Then, I create build folder, where I all cmake like this:

cmake -DCMAKE_TOOLCHAIN_FILE:PATH="../cmake/toolchain.cmake"  ..

CMake output:

-- The CXX compiler identification is GNU 7.3.1
-- Check for working CXX compiler: /home/edo/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu/bin/aarch64-linux-gnu-g++
-- Check for working CXX compiler: /home/edo/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu/bin/aarch64-linux-gnu-g++ -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Found OpenCV: /home/edo/rootfs-xavier/usr (found version "4.1.1") 
-- OpenCV found, version: 4.1.1 in dir /home/edo/rootfs-xavier/usr/include/opencv4
-- GLUT_glut_LIBRARY=GLUT_glut_LIBRARY-NOTFOUND
-- 
-- -------------------------------------------------------------------------------
-- General configuration for aruco 3.1.12
-- -------------------------------------------------------------------------------
-- 
-- Compiler:/home/edo/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu/bin/aarch64-linux-gnu-g++
-- C++ flags (Release):           -O3 -g0  -DNDEBUG
-- C++ flags (Debug):             -O0 -g3  -DDEBUG -D_DEBUG -DPRINT_DEBUG_MESSAGES
-- CMAKE_CXX_FLAGS:         
-- CMAKE_BINARY_DIR:         /home/edo/rootfs-xavier/home/aruco-3.1.12/build
-- 
-- TARGET_PROCESSOR = aarch64
-- BUILD_SHARED_LIBS = ON
-- CMAKE_INSTALL_PREFIX = /usr/local
-- CMAKE_BUILD_TYPE = Debug
-- 
-- BUILD_SVM = 
-- BUILD_UTILS = ON
-- BUILD_TESTS = 
-- BUILD_GLSAMPLES = OFF
-- USE_OWN_EIGEN3=ON
-- OpenCV_DIR=/home/edo/rootfs-xavier/usr/lib/aarch64-linux-gnu/cmake/opencv4
-- 
-- ---------------------------     Documentation     -----------------------------
-- 
-- INSTALL_DOC = OFF
-- USE_LATEX = 
-- USE_DOT = 
-- USE_CHM = 
-- 
-- FOUND OPENGL=NO    LIBS=
-- OpenCV_LIB_DIR= OpenCV_INCLUDE_DIRS=/home/edo/rootfs-xavier/usr/include/opencv4
-- CMAKE_INSTALL_PREFIX=/home/edo/rootfs-xavier/home/aruco-3.1.12/build
-- EIGEN3_INCLUDE_DIR=3rdparty/eigen3
-- USE_TIMERS=OFF
-- 
-- 
-- Change a value with: cmake -D<Variable>=<Value>
-- 
-- Configuring done
-- Generating done
CMake Warning:
  Manually-specified variables were not used by the project:

    CMAKE_TOOLCHAIN_FILE

-- Build files have been written to: /home/edo/rootfs-xavier/home/aruco-3.1.12/build

So, the output looks all right, it seems, that toolchain is properly set and found.
Then I run make -j8:

Scanning dependencies of target aruco
[  1%] Building CXX object src/CMakeFiles/aruco.dir/cameraparameters.cpp.o
[  5%] Building CXX object src/CMakeFiles/aruco.dir/debug.cpp.o
[  5%] Building CXX object src/CMakeFiles/aruco.dir/dictionary.cpp.o
[  6%] Building CXX object src/CMakeFiles/aruco.dir/ippe.cpp.o
[  8%] Building CXX object src/CMakeFiles/aruco.dir/markerdetector.cpp.o
[ 10%] Building CXX object src/CMakeFiles/aruco.dir/markerlabeler.cpp.o
[ 11%] Building CXX object src/CMakeFiles/aruco.dir/posetracker.cpp.o
[ 13%] Building CXX object src/CMakeFiles/aruco.dir/cvdrawingutils.cpp.o
[ 15%] Building CXX object src/CMakeFiles/aruco.dir/dictionary_based.cpp.o
[ 16%] Building CXX object src/CMakeFiles/aruco.dir/marker.cpp.o
[ 18%] Building CXX object src/CMakeFiles/aruco.dir/markerdetector_impl.cpp.o
[ 20%] Building CXX object src/CMakeFiles/aruco.dir/markermap.cpp.o
[ 21%] Building CXX object src/CMakeFiles/aruco.dir/fractaldetector.cpp.o
[ 23%] Building CXX object src/CMakeFiles/aruco.dir/fractallabelers/fractalposetracker.cpp.o
[ 25%] Building CXX object src/CMakeFiles/aruco.dir/fractallabelers/fractalmarkerset.cpp.o
[ 26%] Building CXX object src/CMakeFiles/aruco.dir/fractallabelers/fractalmarker.cpp.o
[ 28%] Building CXX object src/CMakeFiles/aruco.dir/fractallabelers/fractallabeler.cpp.o
[ 30%] Building CXX object src/CMakeFiles/aruco.dir/dcf/dcfmarkermaptracker.cpp.o
[ 31%] Building CXX object src/CMakeFiles/aruco.dir/dcf/dcfmarkertracker.cpp.o
[ 33%] Building CXX object src/CMakeFiles/aruco.dir/dcf/dcf_utils.cpp.o
[ 35%] Building CXX object src/CMakeFiles/aruco.dir/dcf/trackerimpl.cpp.o
[ 36%] Linking CXX shared library libaruco.so
/home/edo/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu/bin/../lib/gcc/aarch64-linux-gnu/7.3.1/../../../../aarch64-linux-gnu/bin/ld: /home/edo/rootfs-xavier/usr/lib/aarch64-linux-gnu/libm.a(e_atan2.o): relocation R_AARCH64_ADR_PREL_PG_HI21 against external symbol `__stack_chk_guard@@GLIBC_2.17' can not be used when making a shared object; recompile with -fPIC
/home/edo/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu/bin/../lib/gcc/aarch64-linux-gnu/7.3.1/../../../../aarch64-linux-gnu/bin/ld: /home/edo/rootfs-xavier/usr/lib/aarch64-linux-gnu/libm.a(e_atan2.o)(.text+0x18): unresolvable R_AARCH64_ADR_PREL_PG_HI21 relocation against symbol `__stack_chk_guard@@GLIBC_2.17'
/home/edo/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu/bin/../lib/gcc/aarch64-linux-gnu/7.3.1/../../../../aarch64-linux-gnu/bin/ld: final link failed: Bad value
collect2: error: ld returned 1 exit status
src/CMakeFiles/aruco.dir/build.make:619: recipe for target 'src/libaruco.so.3.1.12' failed
make[2]: *** [src/libaruco.so.3.1.12] Error 1
CMakeFiles/Makefile2:117: recipe for target 'src/CMakeFiles/aruco.dir/all' failed
make[1]: *** [src/CMakeFiles/aruco.dir/all] Error 2
Makefile:129: recipe for target 'all' failed
make: *** [all] Error 2

It seems, that build itself is working, but then the linking process fails. I also tried setting set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") - then -fPIC flag is also present in cmake output, but the result is the same.

Can anybody please help with this?

Thank you very much!

1 Like
2

I know nothing about your particular software, nor the specifics of fixing this, but some information will help you start…

What you are seeing first is that the instructions for building a dynamically relocatable library is different than creating static content. So building 'src/libaruco.so.3.1.12' failed. Then you see linking to that library fail (because the library was never generated in the first error).

I can only guess, but something is wrong with the cmake configuration and it is trying to build the library with wrong options. Perhaps this is due to some architecture difference, but fundamentally something just seems “off” about the whole configuration. Just as a test, can you build the same thing natively on the Jetson? I suspect the library will again not build correctly, but if you can find out why on a native build this fails, then the answer would likely apply to the cross build environment (and would be easier to solve natively).

1 Like
3

It builds on Jetson platform fine, did it multiple times. I just figured out, what the problem is:

Linker cannot find *.so library files, because symlinks in /usr/local/lib dir are broken. Let me explain. Cross compiling uses root file system of target system, which I got from Xavier NX image provided by Nvidia. Then the whole process is done in target filesystem. What happened is that symlinks in Xavier NX filesystem, which I have located on my laptop (host) HDD under the path /home/user/xavier-filesystem. So the consequent libraries, are located under home/user/xavier-filesystem/usr/local/lib etc. However, symlinks inside xavier filesystem point to locations like /usr/lib instead of /home/user/xavier-filesystem/usr/lib. I think this is the issue here.

I will try to fix it and when I am done with it and it is working, I will mark it as an answer so people who come across this topic can see the solution.

4

You’ve run into one of the often occurring issues of cross compile for user space. Sometimes if your software build knows to look for its architecture (one reason why cross compiling a kernel uses “ARCH=arm64”), then you can mount a clone on the host PC, and then sym link “/usr/lib/aarch64-linux-gnu” to the “<mount point of clone>/usr/lib” and things will “just work” (well, probably not that simple since there are also headers and other content, but that particular issue becomes simple).

I am a fan of setting up full development on the Jetson, and then cloning. If your host PC’s “/usr/lib/aarch64-linux-gnu” is a symbolic link to the correct subdirectory of the clone’s mount point, then you can interchange which environment is running at any moment simply by mounting a different clone.

In cases where there is actual content from cross-development packages already in “/usr/lib/aarch64-linux-gnu”, then you have a few options on how to trick the system into doing what you want without harming the original actual cross-tool content which is part of the host PC and not part of the clone. For example, that content will probably be in subdirectories, and so you simply link a different subdirectory and not the whole aarch64-linux-gnu.

5

Okay, I was able to eliminate mentioned error. I fixed it by converting all symlinks in jetson filesystem to relative links by running symlinks -c path-to-folder-with-symlinks. Now I get like thousand linker errors. Few of them here:

/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu/libgtk-x11-2.0.so.0: undefined reference to `pango_attr_foreground_new'
/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu/libgdk-x11-2.0.so.0: undefined reference to `XFilterEvent'
/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu/libavcodec.so.57: undefined reference to `vorbis_analysis'
/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu/libavcodec.so.57: undefined reference to `vorbis_info_clear'
/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu/libgdk-x11-2.0.so.0: undefined reference to `XSetSelectionOwner'
/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu/libgtk-x11-2.0.so.0: undefined reference to `g_emblemed_icon_get_type'
/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu/libgdk-x11-2.0.so.0: undefined reference to `XMoveWindow'
/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu/libgtk-x11-2.0.so.0: undefined reference to `pango_layout_set_text'

It goes like this. I do not know what to do with this. I tried to add directory paths of the *.so lib files into /home/edo/rootfs-xavier-2/etc/ld.so.conf.d/aarch64-linux-gnu.conf file, but it seems to do nothing (I also run ldconfig within Jetson filesystem using QEMU and chroot).

Any help is appreciated.

6

To illustrate this is one of the errors:

/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu/libgtk-x11-2.0.so.0: undefined reference to `pango_attr_foreground_new'

…each library can link to another library. When you add support for finding a library, then that support must also support anything linked in the chain of libraries. It may be that an option to the cross linker to search more locations. If a library were to link using a relative path to another library, then presumably it would work. If a library were to use an absolute path to find another library, then putting the original library in a new location would cause a failure.

The linker itself can be told which directories it should search when paths don’t exist (when the library is not given an exact path to link to). If you look at “/etc/ld.so.conf.d” for a non-cross linker, then each file might mention a set of directories. Check this output:
egrep -i -H '.*' /etc/ld.so.conf.d/*

If you were to cd to each of those directories, then you would find dynamically relocatable libraries.

If you print all visible linker libraries, then those libraries will be from the set of directories in the previous ld.so.conf.d content:
ldconfig -p

Your linker does not have the path to the other libraries in its path, and so the libraries fulfilling those function signatures will be missing. If you know where those libraries are, then you can add this to your cross linker config. Note that ldconfig -p on your Jetson can be grep’d for a specific library. For example:
ldconfig -p | grep -i 'libgtk-x11'

In my example, on an NX, this shows the library is located at:
/usr/lib/aarch64-linux-gnu/

The linker configuration file “/etc/ld.so.conf.d/aarch64-linux-gnu.conf” names “/usr/lib/aarch64-linux-gnu/”.

Your cross environment could name this if you have directory “/usr/lib/aarch64-linux-gnu/”, and if the libraries are all there, including libgtk-x11-2.0.

The trick is that you are configuring this on your cross linker, not the system linker. You probably have the linker with a name such as “aarch64-linux-gnu-ld” in your cross tool bin directory. This is the linker you must tell about directories to search in.

Assuming your cross linker is “aarch64-linux-gnu-ld”, then if you cd to the bin directory where it exists, you can find out about options you can pass to it:

cd /where/ever/it/is/bin
./aarch64-linux-gnu-ld -h | less

You will find option “-L directory” provides a location to add to the cross linker’s search path in a manner which is similar to what the regular linker searches via “/etc/ld.so.conf.d/*”. This appends a path, and you can use this option more than once. Find out where libraries are at which cannot be found, and edit your Makefile or cmake config to pass “-L somewhere” for each location.

Probably you can also pass that information on the cmake command line or within a config file. Having made your way past the -fPIC issue you are getting further into the build, and now it is time to actually link the libraries…you only need to tell it where to find the libraries.

Incidentally, “-L somewhere” is the same option for the native linker. I could have just said that, but seeing how all of the pieces fit together makes life easier in the long run.

7

I think, that “direct” dependencies (libraries) are fixed by setting absolute paths to relative ones. BUT! What the problem is imho are dependencies of dependencies. Let me explain: In the error output, there is nothing like aruco.so undefined reference to. There is /home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu/libgtk-x11-2.0.so.0: undefined reference to 'pango_attr_foreground_new' . So the variable or whatever object named pango_attr_foreground_new is needed by gtk. On the other hand gtk itself was found by linker. This is quite strange, but it might be related to some pre-saved paths, which were linked/generated at the compilation/install time of gtk (and other aruco dependencies). Does something like this exist? I’m quite sure, that this is the case, because there is not a single case of error saying aruco.so cannot find some library. So we can say, it is runtime error rather than compilation time error?

Thanks for the reply!

8

Note that a symbol won’t care where it is found, but the linker which searches for the symbol will care. The actual failure to find a symbol won’t tell you where it looked. If some command or linker setting causes a relative lookup, or if it causes an absolute path lookup, then eventually, if the library with the symbol is found, there will be no error. Quite often the solution is no more than using multiple “-L” paths to the linker to name more and more library directories until they are all included. Some libraries can actually have a path compiled into them, but that is the exception for individual users in some cases where the user needs to get around a system access issue.

On a regular Linux install I know there is a utility to find required library paths under various tool kits, but I could not tell you the specific case for gtk. Even if I knew this, the tool kit is within a new directory tree as a foreign architecture under the host PC, and not within the native architecture paths. If you were to do research on this I’m pretty sure you’d find something like this for both gtk and QT, but I don’t remember off the top of my brain what that was (someone here can probably give you that information who uses this).

On the other hand, every time you see a missing symbol, and know where just one library is with that content, you could just add a “-L path” to increase where the cross environment linker is looking.

9

Ok, so I had to use rpath-link. I had to edit CMAKE_CXX_FLAGS in top-level CMakelists.txt like this:

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath-link,/path/to/rootfs-xavier/usr/lib/aarch64-linux-gnu -Wl,-rpath-link,/path/to/rootfs-xavier/lib/aarch64-linux-gnu -Wl,-rpath-link,/path/to/rootfs-xavier/usr/lib/aarch64-linux-gnu/tegra")

To sum up, two main issues were present:

  1. I needed to convert absolute symbolic links to relative symbolic links with help of symlinks tool
  2. Linker errors - edit CMakelists.txt like I show here in this post

Thank you for your help!

EDIT: Other things like cmake toolchain settings/config are the same as displayed in this thread.

10

@eduardmraz2, could you post your complete CMakeList.txt ? I am struggling to cross-compile a sample application. I created the following topic:

Thanks.

11

Hey,
my top-level CmakeLists.txt is no different from the aruco original one, only you need to edit CMAKE_CXX_FLAGS. Plus the toolchain.cmake is listed here in this thread. I do not know what exactly you are asking then. Please, read this thread over and try to make steps which I’ve made.

Anyways, here is my CMakeLists.txt, only line 11 is added, cheers.

# ----------------------------------------------------------------------------
#   Basic Configuration
# ----------------------------------------------------------------------------
cmake_minimum_required(VERSION 3.0)
project(aruco VERSION "3.1.12" LANGUAGES CXX)
set(PROJECT_SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}")
set(CMAKE_INCLUDE_CURRENT_DIR ON)
set(CMAKE_CXX_STANDARD 11) # C++11...
set(CMAKE_CXX_STANDARD_REQUIRED ON) #...is required...
set(CMAKE_CXX_EXTENSIONS ON) #...with compiler extensions like gnu++11
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath-link,/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu -Wl,-rpath-link,/home/edo/rootfs-xavier-2/lib/aarch64-linux-gnu -Wl,-rpath-link,/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu/tegra")

include(cmake/options.cmake)

include(cmake/findDependencies.cmake)
include(cmake/compilerOptions.cmake)

add_subdirectory(src)

if(GL_FOUND AND BUILD_GLSAMPLES)
  add_subdirectory(utils_gl)
endif()
if(BUILD_TESTS)
  add_subdirectory(tests)
endif()


if(BUILD_UTILS)
  add_subdirectory(utils)
  add_subdirectory(utils_markermap)
  add_subdirectory(utils_calibration)
  add_subdirectory(utils_dcf)
  IF(BUILD_SVM)
  ADD_SUBDIRECTORY(utils_svm)
  ENDIF()
   ADD_SUBDIRECTORY(utils_fractal)
endif()
include(cmake/printInfo.cmake)
include(cmake/installOptions.cmake)

# Package Generator  #######################################################
IF(BUILD_DEBPACKAGE)
include(cmake/cpack.cmake)
ENDIF()
12

@eduardmraz2, thanks a lot. There are several “include” files in your CMAkeList. Are they Jetson specific or come with cmake? Could you provide those specific to Jetson build? Especially those have settings for CFLAGS and LIB paths.

Thanks.

13

@stuart.xu as I’ve said before: only line added/changed there is this one:

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath-link,/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu -Wl,-rpath-link,/home/edo/rootfs-xavier-2/lib/aarch64-linux-gnu -Wl,-rpath-link,/home/edo/rootfs-xavier-2/usr/lib/aarch64-linux-gnu/tegra")

Then you have to write toolchain.cmake, which is listed here. Nothing too specific about it. You can look those files in ArUco sources.

14

@eduardmraz2, thanks. I will try this out.

1 Like