Crosscompiling

Warning! You are in for a treat! :D - You have been warned. But reward is good, from 3250sec with docker buildx using qemu to 304s on my machine (Clean build)

Instructions

These instructions are made for Ubuntu 20.04.6 LTS

PLEASE DONT RUN ANY SCRIPTS AS SUDO / ROOT! MY SCRIPTS ARE NOT "SAFE"!

Instead give yourself write-access to /opt, sudo chmod 777 /opt/ for example.

 

---

Update source files

• Add [arch=amd64] after deb on each line in /etc/apt/sources.list (This should be changed to your current machines native arch!)
• Add [arch=amd64,arm64] after deb in /etc/apt/sources.list.d/ros-latest.list

• Create /etc/apt/sources.list.d/arm-cross-compile-sources.list

deb [arch=arm64] http://ports.ubuntu.com/ focal main restricted
deb [arch=arm64] http://ports.ubuntu.com/ focal-updates main restricted
deb [arch=arm64] http://ports.ubuntu.com/ focal universe
deb [arch=arm64] http://ports.ubuntu.com/ focal-updates universe
deb [arch=arm64] http://ports.ubuntu.com/ focal multiverse
deb [arch=arm64] http://ports.ubuntu.com/ focal-updates multiverse
deb [arch=arm64] http://ports.ubuntu.com/ focal-backports main restricted universe multiverse

Update and install packages

sudo apt update
sudo dpkg --add-architecture arm64

sudo apt install crossbuild-essential-arm64 parallel

Clone open_mower_ros repository

cd /opt/

git clone https://github.com/ClemensElflein/open_mower_ros
git submodule update --init --recursive
cd /opt/open_mower_ros

Setup sysroot and prepare open_mower_ros

We will use CMAKE_SYSROOT /opt/sysroot_arm64 and /opt/sysroot_arm64_download - change these accordingly in the files for your needs. Total size to download is about 200MB and extracted is ~1.1GB. Change the number of download threads as desired.

Create toolchain.cmake

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_VERSION 1)
set(CMAKE_SYSTEM_PROCESSOR aarch64)

# Specify the cross compiler
set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc)
set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++)

#set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static")
#set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static")

# Specify the target system root
set(CMAKE_SYSROOT /opt/sysroot_arm64)

set(SYSROOT_PATH ${CMAKE_SYSROOT})

unset(CMAKE_SYSTEM_PREFIX_PATH)
list(APPEND CMAKE_SYSTEM_PREFIX_PATH ${CMAKE_SYSROOT}/opt/ros/noetic)

set(CMAKE_PREFIX_PATH ${CMAKE_SYSROOT}/opt/ros/noetic ${CMAKE_SYSROOT}/opt/ros/noetic/share)
set(CMAKE_LIBRARY_PATH ${CMAKE_SYSROOT}/opt/ros/noetic/lib)
set(CMAKE_INCLUDE_PATH ${CMAKE_SYSROOT}/opt/ros/noetic/include)
set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT})

# Search for programs in the build host directories
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)

# For libraries and headers in the target directories
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

set(ENV{PKG_CONFIG_PATH} ${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu/pkgconfig/)

Create 1_setup-sysroot.bash

#!/bin/bash

set -e

SYSROOT=/opt/sysroot_arm64
DOWNLOADDIR=/opt/sysroot_downloads
PARALLELDOWNLOAD=6

#########################

rm -rf $SYSROOT
mkdir -p $SYSROOT
mkdir -p $DOWNLOADDIR

# Link might not be needed? Cant remember :D
ln -s $SYSROOT/usr/lib $SYSROOT/lib

install_package() {
#  for PKG in "${PACKAGES[@]}"; do
    PKG=$1
    URL=$(apt-get download --print-uris $PKG | cut -d\' -f2)
    if [ -n "$URL" ]; then
      echo "Downloading $PKG ($URL)"
      wget -q -nc $URL -P $DOWNLOADDIR && dpkg -x $DOWNLOADDIR/$(basename $URL) $SYSROOT
# --show-progress
#      dpkg -x $DOWNLOADDIR/$(basename $URL) $SYSROOT
      echo "Done installing $PKG"
    else
      echo "$PKG NOT FOUND!"
    fi
#  done
}

export -f install_package
export DOWNLOADDIR
export SYSROOT

# read packages.txt into PACKAGES
while IFS= read -r line; do
    PACKAGES+=("$line")
done < "packages.txt"

parallel -j $PARALLELDOWNLOAD install_package ::: "${PACKAGES[@]}"

echo "Done!"

Download packages.txt # Focal Ubuntu 20.04.6 LTS!

wget https://gist.githubusercontent.com/Damme/3161f92ccadf452b4d5f3eee32da097e/raw/8df0172858e95da76091b04f8492705fc0cf0396/packages.txt -O packages.txt

Create 2-fix-links-cmake.bash

#!/bin/bash

SYSROOT=/opt/sysroot_arm64

echo "Linking some missing library links, broken links /hardcoded to /lib/..."
ln -sfv $SYSROOT/usr/lib/aarch64-linux-gnu/libPocoFoundation.so.62 $SYSROOT/usr/lib/aarch64-linux-gnu/libPocoFoundation.so
ln -sfv $SYSROOT/usr/lib/aarch64-linux-gnu/blas/libblas.so.3 $SYSROOT/usr/lib/aarch64-linux-gnu/libblas.so.3
ln -sfv $SYSROOT/usr/lib/aarch64-linux-gnu/lapack/liblapack.so.3 $SYSROOT/usr/lib/aarch64-linux-gnu/liblapack.so.3

ln -sfv $SYSROOT/lib/aarch64-linux-gnu/librt.so.1 $SYSROOT/usr/lib/aarch64-linux-gnu/librt.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libpthread.so.0 $SYSROOT/usr/lib/aarch64-linux-gnu/libpthread.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libBrokenLocale.so.1 $SYSROOT/usr/lib/aarch64-linux-gnu/libBrokenLocale.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libanl.so.1 $SYSROOT/usr/lib/aarch64-linux-gnu/libanl.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libblkid.so.1.1.0 $SYSROOT/usr/lib/aarch64-linux-gnu/libblkid.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libbz2.so.1.0 $SYSROOT/usr/lib/aarch64-linux-gnu/libbz2.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libexpat.so.1.6.11 $SYSROOT/usr/lib/aarch64-linux-gnu/libexpat.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libgpg-error.so.0 $SYSROOT/usr/lib/aarch64-linux-gnu/libgpg-error.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libkeyutils.so.1 $SYSROOT/usr/lib/aarch64-linux-gnu/libkeyutils.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/liblzma.so.5.2.4 $SYSROOT/usr/lib/aarch64-linux-gnu/liblzma.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libm.so.6 $SYSROOT/usr/lib/aarch64-linux-gnu/libm.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libmount.so.1.1.0 $SYSROOT/usr/lib/aarch64-linux-gnu/libmount.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libnsl.so.1 $SYSROOT/usr/lib/aarch64-linux-gnu/libnsl.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libnss_compat.so.2 $SYSROOT/usr/lib/aarch64-linux-gnu/libnss_compat.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libnss_dns.so.2 $SYSROOT/usr/lib/aarch64-linux-gnu/libnss_dns.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libnss_files.so.2 $SYSROOT/usr/lib/aarch64-linux-gnu/libnss_files.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libnss_hesiod.so.2 $SYSROOT/usr/lib/aarch64-linux-gnu/libnss_hesiod.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libnss_nis.so.2 $SYSROOT/usr/lib/aarch64-linux-gnu/libnss_nis.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libnss_nisplus.so.2 $SYSROOT/usr/lib/aarch64-linux-gnu/libnss_nisplus.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libpcre.so.3 $SYSROOT/usr/lib/aarch64-linux-gnu/libpcre.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libresolv.so.2 $SYSROOT/usr/lib/aarch64-linux-gnu/libresolv.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libthread_db.so.1 $SYSROOT/usr/lib/aarch64-linux-gnu/libthread_db.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libutil.so.1 $SYSROOT/usr/lib/aarch64-linux-gnu/libutil.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libuuid.so.1.3.0 $SYSROOT/usr/lib/aarch64-linux-gnu/libuuid.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libz.so.1.2.11 $SYSROOT/usr/lib/aarch64-linux-gnu/libz.so
ln -sfv $SYSROOT/lib/aarch64-linux-gnu/libdl.so.2 $SYSROOT/usr/lib/aarch64-linux-gnu/libdl.so

echo "Fix cmake files with hardcoded paths.."
find $SYSROOT/opt/ros/noetic/ -name "*.cmake" | xargs -i sh -c 'echo {}; sed -i s@/opt/ros/noetic/lib@\${SYSROOT_PATH}/opt/ros/noetic/lib@g {}'
find $SYSROOT/opt/ros/noetic/ -name "*.cmake" | xargs -i sh -c 'echo {}; sed -i s@PREFIX\ /opt/ros/noetic@PREFIX\ \${SYSROOT_PATH}/opt/ros/noetic@g {}'
find $SYSROOT/opt/ros/noetic/ -name "*.cmake" | xargs -i sh -c 'echo {}; sed -i s@\;/usr/@\;\$\{SYSROOT_PATH\}/usr/@g {}'
find $SYSROOT/opt/ros/noetic/ -name "*.cmake" | xargs -i sh -c 'echo {}; sed -i s@\"/usr/@\"\$\{SYSROOT_PATH\}/usr/@g {}'

Create 3-patch-openmower-cmake.bash

#!/bin/bash

cd src/lib/slic3r_coverage_planner

patch CMakeLists.txt <<'EOF'
@@ -33,7 +33,9 @@ ExternalProject_Add(

         CMAKE_CACHE_ARGS
         SOURCE_SUBDIR src
-        CMAKE_ARGS -DSLIC3R_BUILD_TESTS=OFF
+        CMAKE_ARGS -DSLIC3R_BUILD_TESTS=OFF -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
+               -DCMAKE_EXE_LINKER_FLAGS:STRING=--sysroot=${CMAKE_SYSROOT}\;-L${CMAKE_SYSROOT}/opt/ros/noetic/lib\;-L${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu
+               -DCMAKE_SHARED_LINKER_FLAGS:STRING=--sysroot=${CMAKE_SYSROOT}\;-L${CMAKE_SYSROOT}/opt/ros/noetic/lib\;-L${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu
         BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --config Release
 )
EOF

cd ../../../

cd src/lib/xbot_driver_gps

patch CMakeLists.txt <<'EOF'
@@ -32,6 +32,9 @@ ExternalProject_Add(
         INSTALL_COMMAND cmake -E echo "Skipping install step."

         CMAKE_CACHE_ARGS
+        CMAKE_ARGS -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
+               -DCMAKE_EXE_LINKER_FLAGS:STRING=--sysroot=${CMAKE_SYSROOT}\;-L${CMAKE_SYSROOT}/opt/ros/noetic/lib\;-L${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu
+               -DCMAKE_SHARED_LINKER_FLAGS:STRING=--sysroot=${CMAKE_SYSROOT}\;-L${CMAKE_SYSROOT}/opt/ros/noetic/lib\;-L${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu
         BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --config Release
 )
EOF

cd ../../../

Execute scripts

• run the created scripts:

$ bash 1-setup_sysroot.bash
$ bash 2-fix-links-cmake.bash
$ bash 3-patch-openmower-cmake.bash

Compilation

• build with the following command, change -j1 for number of parallel processes. For some reason catkin_make does not parse toolchain.cmake without forcing it...

$ catkin_make -DCMAKE_TOOLCHAIN_FILE=/opt/open_mower_ros/toolchain.cmake -j1

• check the build with

$ find devel/ -type f -executable |xargs file
devel/_setup_util.py:                                      Python script, ASCII text executable
devel/env.sh:                                              a /usr/bin/env sh script, ASCII text executable
devel/lib/mower_utils/planner_test:                        ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/mower_utils/xbot_pose_converter:                 ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/xesc_driver/xesc_driver_node:                    ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/libxesc_2040_driver.so:                          ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/ntrip_client/ntrip_ros.py:                       Python script, ASCII text executable
devel/lib/mower_simulation/mower_simulation:               ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/libvesc_driver.so:                               ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/mower_comms/mower_comms:                         ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/mower_map/mower_map_service:                     ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/libftc_local_planner.so:                         ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/xbot_positioning/xbot_positioning:               ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/xbot_monitoring/xbot_sensor_example:             ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/xbot_monitoring/xbot_monitoring:                 ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/xbot_monitoring/heatmap_generator:               ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/xbot_driver_gps/driver_gps_node:                 ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/mower_logic/mag_calibration:                     ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/mower_logic/monitoring:                          ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/mower_logic/mower_odometry:                      ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/mower_logic/mower_logic:                         ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/libxesc_driver.so:                               ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/slic3r_coverage_planner/slic3r_coverage_planner: ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked
devel/lib/xbot_remote/xbot_remote:                         ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked

• rsync folder to raspberry pi, note the section below!

TODO/Fix me

• Fix [project]/devel/setup.sh - scripts to use correct paths, maybe have a wrapper for setup.sh that setup some env vars for example CMAKE_PREFIX_PATH to omit sysroot_arm64 path, need to figure out what paths are wrong. Seems _setup_util.py is the script that setup ros enviroment.

# One temporary fix is to create a link on the RPi:
$ ln -s / /opt/sysroot_arm64/

• a script to generate packages.txt

Good luck!