Set up ARM builds with QEMU and Docker

This commit allows us to provide both 32-bit and 64-bit ARM binaries for our users.

.github/workflows/main.yml

@@ -6,7 +6,8 @@ jobs:
   build-and-test:
     strategy:
       matrix:
-        ARCH: [x86_64, i386]
+        ARCH: [x86_64, i386, aarch64, armhf]
+        UPDATE: ["1"]
       fail-fast: false
 
     name: ${{ matrix.ARCH }}
@@ -23,6 +24,9 @@ jobs:
         with:
           submodules: recursive
 
+      - name: Set up QEMU integration for Docker
+        run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
       - name: Build and test AppImage
         run: bash ci/build-in-docker.sh
 

ci/build-in-docker.sh

@@ -1,32 +1,120 @@
 #! /bin/bash
 
-set -exo pipefail
+log_message() {
+    color="$1"
+    shift
+    if [ -t 0 ]; then tput setaf "$color"; fi
+    if [ -t 0 ]; then tput bold; fi
+    echo "$@"
+    if [ -t 0 ]; then tput sgr0; fi
+}
+info() {
+    log_message 2 "[info] $*"
+}
+warning() {
+    log_message 3 "[warning] $*"
+}
+error() {
+    log_message 1 "[error] $*"
+}
+
+if [[ "$ARCH" == "" ]]; then
+    error "Usage: env ARCH=... bash $0"
+    exit 2
+fi
+
+set -euo pipefail
+
+this_dir="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")"
 
 case "$ARCH" in
     x86_64)
-        base_image=debian:latest
+        docker_arch=amd64
         ;;
     i386)
-        base_image=i386/debian:latest
+        docker_arch=i386
+        ;;
+    armhf)
+        docker_arch=arm32v7
+        ;;
+    aarch64)
+        docker_arch=arm64v8
         ;;
     *)
-        echo "Usage: env ARCH=[x86_64|i386] $0"
-        exit 2
+        echo "Unsupported \$ARCH: $ARCH"
+        exit 3
+        ;;
 esac
 
-here="$(readlink -f "$(dirname "$0")")"
-cd "$here"
+# first, we need to build the image
+# we always attempt to build it, it will only be rebuilt if Docker detects changes
+# optionally, we'll pull the base image beforehand
+info "Building Docker image for $ARCH (Docker arch: $docker_arch)"
+
+build_args=()
+if [[ "${UPDATE:-}" == "" ]]; then
+    warning "\$UPDATE not set, base image will not be pulled!"
+else
+    build_args+=("--pull")
+fi
 
 docker_image=linuxdeploy-plugin-qt-build:"$ARCH"
 
-docker build -t "$docker_image" --build-arg base_image="$base_image" .
+docker build \
+    --build-arg ARCH="$ARCH" \
+    --build-arg docker_arch="$docker_arch" \
+    "${build_args[@]}" \
+    -t "$docker_image" \
+    "$this_dir"/docker
 
-if isatty &>/dev/null; then
-    extra_args=("-t")
+docker_run_args=()
+
+# only if there's more than 1G of free space in RAM, we can build in a RAM disk
+if [[ "${GITHUB_ACTIONS:-}" != "" ]]; then
+    warning "Building on GitHub actions, which does not support --tmpfs flag -> building on regular disk"
+elif [[ "$(env LC_ALL=C free -m | grep "Mem:" | awk '{print $4}')" -gt 1024 ]]; then
+    info "Host system has enough free memory -> building in RAM disk"
+    docker_run_args+=(
+        "--tmpfs"
+        "/docker-ramdisk:exec,mode=777"
+    )
+else
+    warning "Host system does not have enough free memory -> building on regular disk"
 fi
 
+if [ -t 1 ]; then
+    # needed on unixoid platforms to properly terminate the docker run process with Ctrl-C
+    docker_run_args+=("-t")
+fi
+
+# fix for https://stackoverflow.com/questions/51195528/rcc-error-in-resource-qrc-cannot-find-file-png
+if [ "${CI:-}" != "" ]; then
+    docker_args+=(
+        "--security-opt"
+        "seccomp:unconfined"
+    )
+fi
+
+uid="${UID:-"$(id -u)"}"
+info "Running build with uid $uid"
+
 run_in_docker() {
-    docker run -e ARCH --rm -i "${extra_args[@]}" --init -w /ws -v "$(readlink -f "$here"/..)":/ws --user "$(id -u)" "$docker_image" "$@"
+    # run the build with the current user to
+    #   a) make sure root is not required for builds
+    #   b) allow the build scripts to "mv" the binaries into the /out directory
+    docker run \
+        --rm \
+        -i \
+        --init \
+        -e GITHUB_RUN_NUMBER \
+        -e ARCH \
+        -e CI \
+        --user "$uid" \
+        "${docker_args[@]}" \
+        -v "$(readlink -f "$this_dir"/..):/ws" \
+        -w /ws \
+        "$docker_image" \
+        "$@"
 }
 
 run_in_docker bash ci/build.sh

ci/build.sh

@@ -9,8 +9,8 @@ if [ "$ARCH" == "" ]; then
 fi
 
 # use RAM disk if possible
-if [ "$CI" == "" ] && [ -d /dev/shm ]; then
-    TEMP_BASE=/dev/shm
+if [ "$CI" == "" ] && [ -d /docker-ramdisk ]; then
+    TEMP_BASE=/docker-ramdisk
 else
     TEMP_BASE=/tmp
 fi
@@ -31,30 +31,24 @@ OLD_CWD="$(readlink -f .)"
 
 pushd "$BUILD_DIR"
 
-if [ "$ARCH" == "i386" ]; then
-    EXTRA_CMAKE_ARGS=("-DCMAKE_TOOLCHAIN_FILE=$REPO_ROOT/cmake/toolchains/i386-linux-gnu.cmake" "-DUSE_SYSTEM_CIMG=OFF")
-fi
-
-cmake "$REPO_ROOT" -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo "${EXTRA_CMAKE_ARGS[@]}" -DBUILD_TESTING=On -DSTATIC_BUILD=On
+cmake "$REPO_ROOT" -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_TESTING=ON -DSTATIC_BUILD=ON
 
 make -j"$(nproc)"
 
 ctest -V --no-tests=error
 
 make install DESTDIR=AppDir
 
-# build patchelf
-"$REPO_ROOT"/ci/build-static-patchelf.sh "$(readlink -f out/)"
-patchelf_path="$(readlink -f out/usr/bin/patchelf)"
-
-# build custom strip
-"$REPO_ROOT"/ci/build-static-binutils.sh "$(readlink -f out/)"
-strip_path="$(readlink -f out/usr/bin/strip)"
+patchelf_path="$(which patchelf)"
+strip_path="$(which strip)"
 
 export UPD_INFO="gh-releases-zsync|linuxdeploy|linuxdeploy-plugin-qt|continuous|linuxdeploy-plugin-qt-$ARCH.AppImage"
 
 wget "https://github.com/TheAssassin/linuxdeploy/releases/download/continuous/linuxdeploy-$ARCH.AppImage"
+# qemu is not happy about the AppImage type 2 magic bytes, so we need to "fix" that
+dd if=/dev/zero bs=1 count=3 seek=8 conv=notrunc of=linuxdeploy-"$ARCH".AppImage
 chmod +x linuxdeploy*.AppImage
+
 ./linuxdeploy-"$ARCH".AppImage --appdir AppDir \
     -d "$REPO_ROOT"/resources/linuxdeploy-plugin-qt.desktop \
     -i "$REPO_ROOT"/resources/linuxdeploy-plugin-qt.svg \

ci/docker/Dockerfile

@@ -0,0 +1,59 @@
+# generic Dockerfile for all architectures
+# used to "cache" prebuilt binaries of tools we use internally and installed dependencies
+# needs to be re-run in CI every time as we cannot store auto-built Docker images due to GitHub's strict quota
+# it will save a lot of time in local development environments, though
+
+ARG docker_arch
+
+# we'll just use Debian as a base image for now, mainly because it produces less headache than Ubuntu with arm
+# a big pro is that they ship an up to date CMake in their stable distribution
+# also, they still provide IA-32 builds for some reason...
+# some people in the AppImage community do not (want to) realize i386 is dead for good
+# we are going to drop i686 in the future!
+FROM ${docker_arch}/debian:stable
+
+# variables that need to be availabe during build and runtime must(!) be repeated after FROM
+ARG ARCH
+
+
+SHELL ["bash", "-x", "-c"]
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y build-essential cmake git gcovr patchelf wget \
+            libmagic-dev libjpeg-dev libpng-dev libboost-filesystem-dev libboost-regex-dev \
+            cimg-dev qtbase5-dev qtdeclarative5-dev-tools qml-module-qtquick2 qtdeclarative5-dev \
+            googletest google-mock nlohmann-json3-dev autoconf libtool nano qtwebengine5-dev gdb && \
+    apt-get autoremove --purge -y && \
+    apt-get clean -y
+
+# install into separate destdir to avoid polluting the $PATH with tools like ld that will break things
+ENV TOOLS_DIR=/tools
+
+COPY install-static-binutils.sh /
+RUN bash /install-static-binutils.sh
+
+COPY install-static-patchelf.sh /
+RUN bash /install-static-patchelf.sh
+
+# make patchelf and strip available in $PATH
+# they are static binaries, so we can just copy them
+RUN cp "$TOOLS_DIR"/usr/bin/patchelf /usr/local/bin && \
+    cp "$TOOLS_DIR"/usr/bin/strip /usr/local/bin
+
+ENV CI=1
+
+# in case AppImageLauncher is installed on the host, this little snippet will make AppImages launch normally
+#RUN echo -e '#! /bin/bash\nset -exo pipefail\nexec "$@"' > /usr/bin/AppImageLauncher && \
+#    chmod +x /usr/bin/AppImageLauncher
+
+# we need to configure some Qt tools, therefore we use /tmp as temporary home
+ENV HOME=/tmp
+
+# make sure all AppImages can run in Docker
+ENV APPIMAGE_EXTRACT_AND_RUN=1
+
+# seems necessary to make test.sh run on qemu/arm
+RUN apt-get update && \
+    apt-get install -y qemu-user-static && \
+    apt-get clean -y

ci/build-static-binutils.sh → ci/docker/install-static-binutils.sh

@@ -3,34 +3,6 @@
 set -e
 set -x
 
-INSTALL_DESTDIR="$1"
-
-if [[ "$INSTALL_DESTDIR" == "" ]]; then
-    echo "Error: build dir $BUILD_DIR does not exist" 1>&2
-    exit 1
-fi
-
-# support cross-compilation for 32-bit ISAs
-case "$ARCH" in
-    "x86_64"|"amd64")
-        ;;
-    "i386"|"i586"|"i686")
-        export CFLAGS="-m32"
-        export CXXFLAGS="-m32"
-        ;;
-    *)
-        echo "Error: unsupported architecture: $ARCH"
-        exit 1
-        ;;
-esac
-
-# use RAM disk if possible
-if [ "$CI" == "" ] && [ -d /dev/shm ]; then
-    TEMP_BASE=/dev/shm
-else
-    TEMP_BASE=/tmp
-fi
-
 cleanup () {
     if [ -d "$BUILD_DIR" ]; then
         rm -rf "$BUILD_DIR"
@@ -55,5 +27,5 @@ make -j "$(nproc)"
 make clean
 make -j "$(nproc)" LDFLAGS="-all-static"
 
-# install into user-specified destdir
-make install DESTDIR="$(readlink -f "$INSTALL_DESTDIR")"
+# install into separate destdir to avoid polluting the $PATH with tools like ld that will break things
+make install DESTDIR="${TOOLS_DIR}"