1 The Cross-Compiler

export PREFIX=</your/build/directory>
export SYSROOT=<your/path/to/centos/filesystem>
export INSTALL_DIR=${PREFIX}/cross
export WRKDIR=<your/working/directory>

Build and install binutils

cd $WRKDIR
wget http://ftpmirror.gnu.org/binutils/binutils-2.24.tar.gz
tar xf binutils-2.27.tar.gz
mkdir build-binutils && cd build-binutils
../binutils-2.27/configure --prefix=$INSTALL_DIR --target=arm-linux-gnueabihf --with-arch=armv7 --with-fpu=vfp --with-float=hard --disable-multilib
make -j 8
make install

Build and install mpc

cd $WRKDIR
wget ftp://gcc.gnu.org/pub/gcc/infrastructure/mpc-1.0.3.tar.gz
tar xvf mpc-1.0.3.tar.gz 
mkdir build-mpc && cd build-mpc
../mpc-1.0.3/configure --prefix=$INSTALL_DIR --target=arm-linux-gnueabihf
make -j 8 && make install

Build and install gcc

cd $WRKDIR
wget http://ftpmirror.gnu.org/gcc/gcc-8.2.0/gcc-8.2.0.tar.gz
tar xf gcc-8.2.0.tar.gz 
mkdir -p build-gcc && cd build-gcc/
../gcc-8.2.0/configure --prefix=$INSTALL_DIR --target=arm-linux-gnueabihf --with-arch=armv7 --with-fpu=vfp --with-float=hard --enable-languages=c,c++ --disable-multilib --with-gmp=/usr --with-mpc=$INSTALL_DIR --with-sysroot=$SYSROOT
make -j 8 && make install

To then use the cross-compiler:

export PATH=${INSTALL_DIR}/bin:$PATH
export LD_LIBRARY_PATH=${INSTALL_DIR}/lib:${INSTALL_DIR}/arm-linux-gnueabihf/lib/:$LD_LIBRARY_PATH

2 Prerequisites, e.g. ATLAS TDAQ dependencies

As an example, a subset of the ATLAS TDAQ software will be compiled. Although these dependencies are specific to the ATLAS TDAQ they could be used as an example on how other packages could be built and installed.

export PREFIX=</your/build/directory>
cd ${PREFIX}
mkdir deps
cd deps

2.1 Build Boost

wget https://dl.bintray.com/boostorg/release/1.69.0/source/boost_1_69_0.tar.gz
tar xvzf boost_1_69_0.tar.gz
cd boost_1_69_0
./bootstrap.sh --prefix=${PREFIX}/deps/installed 

cp $PREFIX/zynq-tdaq-cmake/deps/boost_1_69_0/project-config.jam .

./b2 --prefix=${PREFIX}/deps/installed --with=all --address-model=32 -j8 -toolset=arm-linux-gnueabihf-g++ -threading=multi install -s SYSROOT=$SYSROOT

2.2 Build CppUnit

wget http://dev-www.libreoffice.org/src/cppunit-1.14.0.tar.gz
tar -xzf cppunit-1.14.0.tar.gz
cd cppunit-1.14.0/
./configure --build arm CC=${PREFIX}/cross/bin/arm-linux-gnueabihf-gcc CXX=${PREFIX}/cross/bin/arm-linux-gnueabihf-g++ --prefix=${PREFIX}/deps/installed --host arm-linux
make check
make install 

2.3 Build TBB

wget http://lcgpackages.web.cern.ch/lcgpackages/tarFiles/sources/tbb2019_U1.tar.gz
tar -xzf tbb2019_U1.tar.gz
cd tbb2019_U1
make arch=armv7 compilier=arm-linux-gnueabihf-gcc

2.4 Build i2c-tools

wget https://mirrors.edge.kernel.org/pub/software/utils/i2c-tools/i2c-tools-4.1.tar.gz
tar -xzf i2c-tools-4.1.tar.gz
cd i2c-tools-4.1
cp $PREFIX/zynq-tdaq-cmake/deps/i2c-tools-4.1/Makefile .
export DESTDIR=$SYSROOT/usr
export PREFIX2=$PREFIX
make
sudo -E bash -c 'make install'
unset DESTDIR
unset PREFIX2

3 ATLAS TDAQ (Example of using CMake with Cross-Compilation)

This is an example on how to build and install the ATLAS TDAQ software. It is specific to ATLAS, but could be used as an example on how to use CMake with cross-compilation.

3.1 Checkout necessary packages

export PATH=${PREFIX}/zynq-tdaq-cmake/checkout_release:${PATH}
export CHECKOUT_AREA=${PREFIX}
git clone https://:@gitlab.cern.ch:8443/atlas-l1ct/zynq-tdaq-cmake.git -b muctpi_tdaq8
git clone https://:@gitlab.cern.ch:8443/atlas-tdaq-software/cmake_tdaq.git

checkout_release tdaq-common tdaq-common-03-03-00 ${PREFIX}/zynq-tdaq-cmake/checkout_release/tdaq-common-packages.txt
checkout_release tdaq tdaq-08-02-01 ${PREFIX}/zynq-tdaq-cmake/checkout_release/tdaq-packages.txt
cd tdaq/tdaq-08-02-01
#copy & patch
cp ${PREFIX}/zynq-tdaq-cmake/tdaq/CMakeLists.txt ${PREFIX}/tdaq/tdaq-08-02-01/
cp ${PREFIX}/zynq-tdaq-cmake/tdaq-common/CMakeLists.txt ${PREFIX}/tdaq-common/tdaq-common-03-03-00/

3.2 Build tdaq-common

cd $PREFIX
export CMAKE_PREFIX_PATH=${PREFIX}/cmake_tdaq:${PREFIX}:${CMAKE_PREFIX_PATH}
export LD_LIBRARY_PATH=${SYSROOT}/lib:${SYSROOT}/usr/lib:${LD_LIBRARY_PATH}
source $PREFIX/zynq-tdaq-cmake/l1ct/setup.sh
export RPATH_FLAG="-Wl,-rpath-link,${PREFIX}/cross/arm-linux-gnueabihf/lib/:${SYSROOT}/usr/lib:${PREFIX}/deps/installed/lib:${PREFIX}/deps/tbb-2019_U1/build/linux_armv7_gcc_cc4.8.5_libc2.17_kernel3.10.0_release"

cd tdaq-common/tdaq-common-03-03-00
mkdir -p build && cd build

cmake3 -DBINARY_TAG=armv7-xlnx-gcc8-opt \
	-DCMAKE_TOOLCHAIN_FILE=${PREFIX}/zynq-tdaq-cmake/l1ct/toolchain.cmake \
	-DBoost_LIBRARY_DIR_RELEASE=${PREFIX}/deps/installed/lib/ \
	-DBoost_INCLUDE_DIR=${PREFIX}/deps/installed/include ../
make install
#We copy jar files from cvmfs so that we don't have to build them
cp -r /cvmfs/atlas.cern.ch/repo/sw/tdaq/tdaq/tdaq-08-02-01/installed/share/lib/ ${PREFIX}/tdaq-common/tdaq-common-03-03-00/installed/share

3.3 Build tdaq

export LD_LIBRARY_PATH=${PREFIX}/deps/installed/lib/:${LD_LIBRARY_PATH}
cd $PREFIX
export CMAKE_PREFIX_PATH=${PREFIX}/cmake_tdaq:${PREFIX}:${CMAKE_PREFIX_PATH}
export LD_LIBRARY_PATH=${SYSROOT}/lib:${SYSROOT}/usr/lib:${LD_LIBRARY_PATH}
export RPATH_FLAG="-Wl,-rpath-link,${PREFIX}/cross/arm-linux-gnueabihf/lib/:${SYSROOT}/usr/lib:${PREFIX}/deps/installed/lib:${PREFIX}/deps/tbb-2019_U1/build/linux_armv7_gcc_cc4.8.5_libc2.17_kernel3.10.0_release"
${PREFIX}/deps/tbb-2019_U1/build/linux_armv7_gcc_cc4.8.5_libc2.17_kernel3.10.0_release/tbbvars.sh
source $PREFIX/zynq-tdaq-cmake/l1ct/setup.sh

export PATH=${PREFIX}/zynq-tdaq-cmake/checkout_release:${PATH}
checkout_release tdaq tdaq-08-02-01 ${PREFIX}/zynq-tdaq-cmake/checkout_release/tdaq-packages.txt

cd tdaq/tdaq-08-02-01/
git clone https://:@gitlab.cern.ch:8443/atlas-l1ct/muctpi-test-runcontrol.git

mkdir -p build && cd build
source ${PREFIX}/deps/tbb-2019_U1/build/linux_armv7_gcc_cc4.8.5_libc2.17_kernel3.10.0_release/tbbvars.sh

cmake3 -DBINARY_TAG=armv7-xlnx-gcc8-opt \
-DTBB_INCLUDE_DIR=${PREFIX}/deps/tbb-2019_U1/include \
-DTBB_ROOT_DIR=${PREFIX}/deps/tbb-2019_U1/build/linux_armv7_gcc_cc4.8.5_libc2.17_kernel3.10.0_release \
-DBoost_DIR=${PREFIX}/deps/installed/ \
-DBoost_LIBRARY_DIR_RELEASE=${PREFIX}/deps/installed/lib/ \
-DBoost_INCLUDE_DIR=${PREFIX}/deps/installed/include \
-DCPPUNIT_INCLUDE_DIR=${PREFIX}/deps/installed/include \
-DCPPUNIT_LIBRARY=${PREFIX}/deps/installed/lib/libcppunit.a \
-DJAVA_HOME=/cvmfs/sft.cern.ch/lcg/releases/java/8u91-ae32f/x86_64-centos7-gcc8-opt \
-DJAVA_AWT_LIBRARY=/cvmfs/sft.cern.ch/lcg/releases/java/8u91-ae32f/x86_64-centos7-gcc8-opt/include \
-DJAVA_JVM_LIBRARY=/cvmfs/sft.cern.ch/lcg/releases/java/8u91-ae32f/x86_64-centos7-gcc8-opt/include/linux/include \
-DJAVA_INCLUDE_PATH=/cvmfs/sft.cern.ch/lcg/releases/java/8u91-ae32f/x86_64-centos7-gcc8-opt/include \
-DJAVA_INCLUDE_PATH2=/cvmfs/sft.cern.ch/lcg/releases/java/8u91-ae32f/x86_64-centos7-gcc8-opt/include/linux \
-DJAVA_AWT_INCLUDE_PATH=/cvmfs/sft.cern.ch/lcg/releases/java/8u91-ae32f/x86_64-centos7-gcc8-opt/lib/amd64 \
-DJAVA_INCLUDE_DIR=/cvmfs/sft.cern.ch/lcg/releases/java/8u91-ae32f/x86_64-centos7-gcc8-opt/include/ \
-DCMAKE_TOOLCHAIN_FILE=${PREFIX}/zynq-tdaq-cmake/l1ct/toolchain.cmake \
-DTDAQ_CLASSPATH=/cvmfs/atlas.cern.ch/repo/sw/tdaq/tdaq/tdaq-08-02-01/installed/share/lib/* \
-DTDAQ_INST_PATH=/cvmfs/atlas.cern.ch/repo/sw/tdaq/tdaq/tdaq-08-02-01/installed/ \
-Dtdaq-common_DIR=${PREFIX}/tdaq-common/tdaq-common-03-03-00/installed/share/cmake/tdaq-common/armv7-xlnx-gcc8-opt/ ..

make
make install

4 ATLAS TDAQ Run Control Application (Example of using Cross-Compiled Software)

The following instructions are specific to the ATLAS TDAQ software, but they could be used as an example on how other cross-compiled software could be used on the client.

The tdaq software is mounted via nfs to the client.

export NFS_PREFIX=/location/in/respect/to/client/

On the Client add the user and the group that your you are using on the host

useradd -u <user_id> -g <group id> <your_host_user>
groupadd <your_host_group>

Then add rsa authentication to the client.

ssh-copy-id -i ~/.ssh/id_rsa.pub user@<client_ip>

4.1 Setup

git clone ssh://git@gitlab.cern.ch:7999/atlas-l1ct/muctpi-test-partition.git -b tdaq-8
export TDAQ_INST_PATH=/cvmfs/atlas.cern.ch/repo/sw/tdaq/tdaq/tdaq-08-02-01/installed/
export CMTCONFIG=x86_64-centos7-gcc8-opt
source ${TDAQ_INST_PATH}/setup.sh

source ./setup.sh
touch ${PREFIX}/ipc_root.ref
export TDAQ_IPC_INIT_REF=file:${PREFIX}/ipc_root.ref
chmod +777 ${PREFIX}/ipc_root.ref

4.2 Edit database

On the host:

dbe -f Training/partitions/part_training.data.xml

1. Add Host and Client as Computers
2. Edit Partition initial and part_training "DefaultHost" to read your host's name
3. Edit Segment MyDetector "Hosts" to read your host name
4. Edit MuctpiRCApplication muctpi_rc_app "RunsOn" to read your client's name
5. Edit Partition part_training "LogRoot" to read to your log directory
6. Create SW_Repository BOOST_EXTERNAL_Zynq with "installationPath": ${NFS_PREFIX}/deps/installed/lib/armv7-xlnx-gcc8-opt/lib .

4.3 Add symlinks

On the client:

mkdir -p ${PREFIX}
ln -s ${NFS_PREFIX}/ipc_root.ref ${PREFIX}/ipc_root.ref

cd ${NFS_PREFIX}/deps/installed/lib/
mkdir armv7-xlnx-gcc8-opt
cd armv7-xlnx-gcc8-opt
ln -s ../ lib

cd /usr/lib/
mkdir armv7-xlnx-gcc8-opt
cd armv7-xlnx-gcc8-opt
ln -s ../ lib

cd /usr/local/lib
mkdir armv7-xlnx-gcc8-opt
cd armv7-xlnx-gcc8-opt
ln -s ../ lib

cd ${NFS_PREFIX}/deps/tbb-2019_U1/build/linux_armv7_gcc_cc4.8.5_libc2.17_kernel3.10.0_release
mkdir -p lib/armv7-xlnx-gcc8-opt
cd lib/armv7-xlnx-gcc8-opt
ln -s ../../ lib

4.4 Run the application

On the host:

setup_daq daq/segments/setup-initial.data.xml initial -ng
setup_daq -p part_training -d Training/partitions/part_training.data.xml

-- PanagiotisPapageorgiou - 2019-07-04