fix: Cross-compilation to ARM64 (#4142)

* feat(docker/docs): explain how building lemmy works

Signed-off-by: Enzo NOCERA <enzo@nocera.eu>

* feat: add arm build

* review: rename script & fix typo

* feat(ci): allow cross platform compilation

* feat(ci): prettier

* fix(docker): fix base image name

* fix: add dockerfile in CI path

Signed-off-by: Enzo Nocera <enzo@nocera.eu>

* fix(docker): fix runner name

* fix(docker): fix builder base image

* fix(docker): fix builder base image platform

* fix(docker): avoid using the wrapper adduser/addgroup

* feat: avoid adding the whole docker directory in the build context

---------

Signed-off-by: Enzo NOCERA <enzo@nocera.eu>
Signed-off-by: Enzo Nocera <enzo@nocera.eu>
Co-authored-by: Dessalines <dessalines@users.noreply.github.com>
Co-authored-by: Nutomic <me@nutomic.com>

.dockerignore

@@ -1,6 +1,7 @@
 # build folders and similar which are not needed for the docker build
 target
 docker
+!docker/builders/lemmy-builder-arm64/docker-build.sh
 api_tests
 ansible
 tests

.woodpecker.yml

@@ -227,17 +227,29 @@ steps:
       - event: push
         branch: main
 
+  publish_builder_arm64:
+    image: woodpeckerci/plugin-docker-buildx
+    secrets: [docker_username, docker_password]
+    settings:
+      repo: dessalines/lemmy-builder-arm64
+      dockerfile: docker/builders/lemmy-builder-arm64/Dockerfile
+      platforms: linux/amd64
+      build_args:
+        - RUST_RELEASE_MODE=release
+      tag: ${CI_COMMIT_TAG}
+    when:
+      event: tag
+
   publish_release_docker:
     image: woodpeckerci/plugin-docker-buildx
     secrets: [docker_username, docker_password]
     settings:
       repo: dessalines/lemmy
       dockerfile: docker/Dockerfile
-      # TODO fix arm build: see: https://woodpecker.join-lemmy.org/repos/129/pipeline/2888/20
+      platforms: linux/amd64, linux/arm64
-      # platforms: linux/amd64,linux/arm64
-      platforms: linux/amd64
       build_args:
         - RUST_RELEASE_MODE=release
+        - LEMMY_VERSION=${CI_COMMIT_TAG}
       tag: ${CI_COMMIT_TAG}
     when:
       event: tag

docker/Dockerfile

@@ -1,10 +1,18 @@
+# syntax=docker/dockerfile:1.6
 ARG RUST_VERSION=1.72.1
 ARG CARGO_BUILD_FEATURES=default
 ARG RUST_RELEASE_MODE=debug
+ARG LEMMY_VERSION="dev"
+
 ARG AMD_BUILDER_IMAGE=rust:${RUST_VERSION}
-ARG ARM_BUILDER_IMAGE=blackdex/rust-musl:aarch64-musl-stable-${RUST_VERSION}-openssl3
+ARG ARM_BUILDER_IMAGE="dessalines/lemmy-builder-arm64:${LEMMY_VERSION}"
+
 ARG AMD_RUNNER_IMAGE=debian:bookworm-slim
-ARG ARM_RUNNER_IMAGE=alpine:3.18
+ARG ARM_RUNNER_IMAGE=debian:bookworm-slim
+
+ARG UNAME=lemmy
+ARG UID=1000
+ARG GID=1000
 
 # AMD64 builder
 FROM --platform=${BUILDPLATFORM} ${AMD_BUILDER_IMAGE} AS build-amd64
@@ -33,79 +41,41 @@ RUN set -ex; \
     fi
 
 # ARM64 builder
-# TODO currently broken
+# NB(raskyld): this is a hack to be able to COPY --from= this image, because the variable doesn't
-# FROM --platform=${BUILDPLATFORM} ${ARM_BUILDER_IMAGE} as build-arm64
+# seem to be expended in --form arg of COPY :(
+FROM --platform=linux/amd64 ${ARM_BUILDER_IMAGE} AS build-arm64
 
-# ENV DEBIAN_FRONTEND=noninteractive
+# amd64 base runner
-# ENV CARGO_HOME=/root/.cargo
+FROM ${AMD_RUNNER_IMAGE} AS runner-linux-amd64
-# ENV PQ_LIB_DIR=/usr/local/musl/pq15/lib
-
-# RUN apt update && apt install -y \
-#     --no-install-recommends \
-#     git
-
-# RUN mkdir -pv "${CARGO_HOME}" && \
-#     rustup set profile minimal && \
-#     rustup target add aarch64-unknown-linux-musl
-
-# ARG CARGO_BUILD_FEATURES
-# ARG RUST_RELEASE_MODE
-
-# WORKDIR /lemmy
-
-# COPY . ./
-
-# # Debug build
-# RUN --mount=type=cache,target=/lemmy/target set -ex; \
-#     if [ "${RUST_RELEASE_MODE}" = "debug" ]; then \
-#         echo "pub const VERSION: &str = \"$(git describe --tag)\";" > crates/utils/src/version.rs; \
-#         cargo build --target=aarch64-unknown-linux-musl --features "${CARGO_BUILD_FEATURES}"; \
-#         mv target/aarch64-unknown-linux-musl/debug/lemmy_server ./lemmy; \
-#     fi
-
-# # Release build
-# RUN set -ex; \
-#     if [ "${RUST_RELEASE_MODE}" = "release" ]; then \
-#         echo "pub const VERSION: &str = \"$(git describe --tag)\";" > crates/utils/src/version.rs; \
-#         cargo build --target=aarch64-unknown-linux-musl --features "${CARGO_BUILD_FEATURES}" --release; \
-#         mv target/aarch64-unknown-linux-musl/release/lemmy_server ./lemmy; \
-#     fi
-
-## Final image
-FROM ${AMD_RUNNER_IMAGE}
 
 # Federation needs CA certificates
 RUN apt update && apt install -y libssl-dev libpq-dev ca-certificates
 
-# Debian / Ubuntu non-root user creds
+COPY --from=build-amd64 --chmod=0755 /lemmy/lemmy /usr/local/bin
-ARG UNAME=lemmy
+
-ARG UID=1000
+# arm base runner
-ARG GID=1000
+FROM ${ARM_RUNNER_IMAGE} AS runner-linux-arm64
-RUN groupadd -g $GID -o $UNAME
+
-RUN useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME
+RUN apt update && apt install -y ca-certificates libssl-dev libpq-dev
+
+COPY --from=build-arm64 --chmod=0755 /home/lemmy/lemmy_server /usr/local/bin
+
+# Final image that use a base runner based on the target OS and ARCH
+FROM runner-${TARGETOS}-${TARGETARCH}
+
+LABEL org.opencontainers.image.authors="The Lemmy Authors"
+LABEL org.opencontainers.image.source="https://github.com/LemmyNet/lemmy"
+LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later"
+LABEL org.opencontainers.image.description="A link aggregator and forum for the fediverse"
+
+ARG UNAME
+ARG GID
+ARG UID
+
+RUN groupadd -g ${GID} -o ${UNAME} && \
+    useradd -m -u ${UID} -g ${GID} -o -s /bin/bash ${UNAME}
 USER $UNAME
 
-COPY --from=build-amd64 /lemmy/lemmy ./
+ENTRYPOINT ["lemmy_server"]
-CMD ["./lemmy"]
 EXPOSE 8536
 STOPSIGNAL SIGTERM
-
-## Arm Runner
-# FROM --platform=${BUILDPLATFORM} ${ARM_RUNNER_IMAGE}
-
-# ARG UNAME=lemmy
-# ARG UID=1000
-# ARG GID=1000
-
-# RUN apk add --no-cache ca-certificates
-
-# COPY --from=build-arm64 --chmod=0755 /lemmy/lemmy /usr/local/bin
-
-# RUN addgroup -S -g ${GID} ${UNAME} && \
-#     adduser -S -H -D -G ${UNAME} -u ${UID} -g "" -s /sbin/nologin ${UNAME}
-# USER $UNAME
-
-# CMD ["lemmy"]
-# EXPOSE 8536
-# STOPSIGNAL SIGTERM
-

docker/README.md

@@ -0,0 +1,21 @@
+# Building Lemmy Images
+
+Lemmy's images are meant to be **built** on `linux/amd64`,
+but they can be **executed** on both `linux/amd64` and `linux/arm64`.
+
+To do so we need to use a _cross toolchain_ whose goal is to build
+**from** amd64 **to** arm64.
+
+Namely, we need to link the _lemmy_server_ with `pq` and `openssl`
+shared libraries and a few others, and they need to be in `arm64`,
+indeed.
+
+The toolchain we use to cross-compile is specifically tailored for
+Lemmy's needs, see [the image repository][image-repo].
+
+#### References
+
+- [The Linux Documentation Project on Shared Libraries][tldp-lib]
+
+[tldp-lib]: https://tldp.org/HOWTO/Program-Library-HOWTO/shared-libraries.html
+[image-repo]: https://github.com/raskyld/lemmy-cross-toolchains

docker/builders/lemmy-builder-arm64/Dockerfile

@@ -0,0 +1,23 @@
+ARG ARM_CROSS_TOOLCHAIN="ghcr.io/raskyld/aarch64-lemmy-linux-gnu:v0.1.0"
+
+FROM ${ARM_CROSS_TOOLCHAIN}
+
+# NB(raskyld): Please, do not hesitate to contact me through @raskyld@social.vivaldi.net (mastodon)
+# If you have any question about the cross-toolchain
+LABEL org.opencontainers.image.authors="Enzo Nocera <enzo@nocera.eu>"
+LABEL org.opencontainers.image.source="https://github.com/LemmyNet/lemmy"
+LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later"
+LABEL org.opencontainers.image.description="A prebuilt lemmy server using a cross toolchain from amd64 to aarch64/arm64"
+
+ARG CARGO_BUILD_FEATURES=default
+ARG RUST_RELEASE_MODE=debug
+
+WORKDIR /home/lemmy/src
+USER 10001:10001
+
+COPY --chown=lemmy:lemmy . ./
+
+ENV RUST_RELEASE_MODE=${RUST_RELEASE_MODE} \
+    CARGO_BUILD_FEATURES=${CARGO_BUILD_FEATURES}
+
+RUN --mount=type=cache,target=./target,uid=10001,gid=10001 bash ./docker/builders/lemmy-builder-arm64/docker-build.sh

docker/builders/lemmy-builder-arm64/docker-build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -e;
+
+source "$HOME/.cargo/env"
+
+case "$RUST_RELEASE_MODE" in
+    "debug")
+        echo "pub const VERSION: &str = \"$(git describe --tag)\";" > "crates/utils/src/version.rs"
+        cargo build --features "${CARGO_BUILD_FEATURES}"
+        cp "./target/$CARGO_BUILD_TARGET/$RUST_RELEASE_MODE/lemmy_server" /home/lemmy/lemmy_server
+        ;;
+    "release")
+        # Pass a value to $USE_RELEASE_CACHE to avoid purging the cache for release builds
+        [[ -z "$USE_RELEASE_CACHE" ]] || cargo clean --release
+        echo "pub const VERSION: &str = \"$(git describe --tag)\";" > "crates/utils/src/version.rs"
+        cargo build --features "${CARGO_BUILD_FEATURES}" --release
+        cp "./target/$CARGO_BUILD_TARGET/$RUST_RELEASE_MODE/lemmy_server" /home/lemmy/lemmy_server
+        ;;
+esac