fixes, simplification, cross-lang LTO

This commit is contained in:
Jade Ellis 2025-01-18 22:46:22 +00:00
parent d000de396c
commit 55ee85a17c
No known key found for this signature in database
GPG key ID: 8705A2A3EBF77BD2

View file

@ -1,83 +1,56 @@
FROM --platform=$BUILDPLATFORM docker.io/tonistiigi/xx AS xx
FROM --platform=$BUILDPLATFORM rust:1-slim-bookworm AS base
FROM --platform=$BUILDPLATFORM rust:1-slim-bookworm AS builder
ARG BUILDPLATFORM
# Don't delete the apt cache
RUN rm -f /etc/apt/apt.conf.d/docker-clean
FROM --platform=$BUILDPLATFORM rust:latest AS builder
ARG LLVM_VERSION=19
# Install repo tools
# Line one: compiler tools
# Line two: curl, for downloading binaries
# Line three: for xx-verify
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
apt-get update && apt-get install -y \
clang-${LLVM_VERSION} lld-${LLVM_VERSION} pkg-config make \
curl \
file
# Create symlinks for clang
RUN ln -s /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang
RUN ln -s /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang++
RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/lld
# Developer tool versions
# renovate: datasource=github-releases depName=cargo-bins/cargo-binstall
ENV BINSTALL_VERSION=1.10.21
# renovate: datasource=github-releases depName=psastras/sbom-rs
ENV CARGO_SBOM_VERSION=0.9.1
# renovate: datasource=crate depName=lddtree
ENV LDDTREE_VERSION=0.3.7
RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
RUN cargo binstall --no-confirm cargo-sbom --version $CARGO_SBOM_VERSION
RUN cargo binstall --no-confirm lddtree --version $LDDTREE_VERSION
# Set up xx (cross-compilation scripts)
COPY --from=xx / /
ARG TARGETPLATFORM
# Determine target platform identifiers
RUN RUST_TARGET=$(case $TARGETPLATFORM in \
"linux/amd64") echo x86_64-unknown-linux-gnu ;; \
"linux/arm64") echo aarch64-unknown-linux-gnu ;; \
*) echo "Unsupported target platform $TARGETPLATFORM" 1>&2; exit 1 ;; \
esac) && \
echo "RUST_TARGET=$RUST_TARGET" >> /etc/environment
RUN DEB_ARCH=$(case $TARGETPLATFORM in \
"linux/amd64") echo amd64 ;; \
"linux/arm64") echo arm64 ;; \
*) exit 1 ;; \
esac) && \
echo "DEB_ARCH=$DEB_ARCH" >> /etc/environment
# install libraries linked by the binary
# xx-* are xx-specific meta-packages
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
xx-apt-get install -y \
xx-c-essentials xx-cxx-essentials \
liburing-dev
RUN GCC_TARGET=$(case $TARGETPLATFORM in \
"linux/amd64") echo x86-64-linux-gnu ;; \
"linux/arm64") echo aarch64-linux-gnu ;; \
*) exit 1 ;; \
esac) && \
echo "GCC_TARGET=$GCC_TARGET" >> /etc/environment
# install build-time deps
RUN set -o allexport && \
. /etc/environment && \
dpkg --add-architecture $DEB_ARCH && \
apt-get update && apt-get install -y --no-install-recommends \
lld \
gcc-$GCC_TARGET g++-$GCC_TARGET \
libc6-dev-$DEB_ARCH-cross \
libclang-dev liburing-dev \
liburing-dev:$DEB_ARCH && \
rm -rf /var/lib/apt/lists/*
# set linkers, compilers and pkg-config libdir
RUN VARS=$(case $TARGETPLATFORM in \
"linux/amd64") \
echo "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc" && \
echo "CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc" && \
echo "CXX_x86_64_unknown_linux_gnu=\"x86_64-linux-gnu-g++\"" && \
echo "PKG_CONFIG_LIBDIR=/usr/lib/x86_64-linux-gnu/pkgconfig" \
;; \
"linux/arm64") \
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" && \
echo "CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc" && \
echo "CXX_aarch64_unknown_linux_gnu=\"aarch64-linux-gnu-g++\"" && \
echo "PKG_CONFIG_LIBDIR=/usr/lib/aarch64-linux-gnu/pkgconfig" \
;; \
*) exit 1 ;; \
esac) && \
echo "$VARS" >> /etc/environment
# enable cross-platform linking of libraries
RUN echo "PKG_CONFIG_ALLOW_CROSS=true" >> /etc/environment
ARG CONDUWUIT_VERSION_EXTRA=
ENV CONDUWUIT_VERSION_EXTRA=$CONDUWUIT_VERSION_EXTRA
# Set up Rust toolchain
WORKDIR /app
COPY ./rust-toolchain.toml .
RUN rustc --version
RUN set -o allexport && \
. /etc/environment && \
rustup target add $RUST_TARGET
# Developer tool versions
# renovate: datasource=github-releases depName=cargo-binstall packageName=cargo-bins/cargo-binstall
ENV BINSTALL_VERSION=1.10.17
# renovate: github-releases depName=cargo-sbom packageName=psastras/sbom-rs
ENV CARGO_SBOM_VERSION=0.9.1
RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
RUN cargo binstall --no-confirm cargo-sbom --version $CARGO_SBOM_VERSION
RUN rustup target add $(xx-cargo --print-target-triple)
# Get source
COPY . .
@ -86,36 +59,51 @@ COPY . .
# We disable incremental compilation to save disk space, as it only produces a minimal speedup for this case.
ENV CARGO_INCREMENTAL=0
# Configure pkg-config
RUN echo "PKG_CONFIG_LIBDIR=/usr/lib/$(xx-info)/pkgconfig" >> /etc/environment
RUN echo "PKG_CONFIG=/usr/bin/$(xx-info)-pkg-config"
RUN echo "PKG_CONFIG_ALLOW_CROSS=true" >> /etc/environment
# Configure cc to use clang version
RUN echo "CC=clang" >> /etc/environment
RUN echo "CXX=clang++" >> /etc/environment
# Cross-language LTO
RUN echo "CFLAGS=-flto" >> /etc/environment
RUN echo "CXXFLAGS=-flto" >> /etc/environment
RUN echo "RUSTFLAGS='-Clinker-plugin-lto -Clinker=clang -Clink-arg=-fuse-ld=lld'"
# CPU specific optimizations
ARG TARGET_CPU=
RUN if [ -n "${TARGET_CPU}" ]; then \
echo "CFLAGS='-march=${TARGET_CPU}'" >> /etc/environment && \
echo "CXXFLAGS='-march=${TARGET_CPU}'" >> /etc/environment && \
echo "RUSTFLAGS='-C target-cpu=${TARGET_CPU}'" >> /etc/environment; \
RUN set -o allexport && \
. /etc/environment && \
if [ -n "${TARGET_CPU}" ]; then \
echo "CFLAGS='${CFLAGS} -march=${TARGET_CPU}'" >> /etc/environment && \
echo "CXXFLAGS='${CCCFLAGS} -march=${TARGET_CPU}'" >> /etc/environment && \
echo "RUSTFLAGS='${RUSTFLAGS} -C target-cpu=${TARGET_CPU}'" >> /etc/environment; \
fi
# Conduwuit specific variable
ARG CONDUWUIT_VERSION_EXTRA=
ENV CONDUWUIT_VERSION_EXTRA=$CONDUWUIT_VERSION_EXTRA
RUN cat /etc/environment
RUN mkdir /out
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git/db \
--mount=type=cache,target=/app/target \
set -o allexport && \
. /etc/environment && \
cargo build --locked --release --target $RUST_TARGET && \
cp ./target/$RUST_TARGET/release/conduwuit /out/app
xx-cargo build --locked --release && \
xx-verify ./target/$(xx-cargo --print-target-triple)/release/conduwuit && \
cp ./target/$(xx-cargo --print-target-triple)/release/conduwuit /out/app
RUN cargo sbom > /out/sbom.spdx.json
# find dynamically linked dependencies
RUN mkdir /out/libs \
&& ldd /out/app | grep '=>' | awk '{print $(NF-1)}' | xargs -I {} cp {} /out/libs/
# libraries with a hardcoded path, like ld
# (see for example https://github.com/vlang/v/issues/8682)
# Excluding linux-vdso.so, as it is a part of the kernel
RUN mkdir /out/libs-root \
&& ldd /out/app | grep -v '=>' | grep -v 'linux-vdso.so' | awk '{print $(NF-1)}' | xargs -I {} install -D {} /out/libs-root{}
# RUN ldd /out/app
# ldd /out/app | grep -v 'linux-vdso.so' | awk '{print $(NF-1)}'
# RUN ls /libs
&& lddtree /out/app | awk '{print $(NF-0) " " $1}' | sort -u -k 1,1 | awk '{print "install", "-D", $1, "/out/libs" (($2 ~ /^\//) ? $2 : $1)}' | xargs -I {} sh -c {}
FROM scratch
@ -124,19 +112,14 @@ WORKDIR /
# Copy root certs for tls into image
# You can also mount the certs from the host
# --volume /etc/ssl/certs:/etc/ssl/certs:ro
COPY --from=rust:latest /etc/ssl/certs /etc/ssl/certs
COPY --from=base /etc/ssl/certs /etc/ssl/certs
# Copy our build
COPY --from=builder /out/app ./app
# Copy SBOM
COPY --from=builder /out/sbom.spdx.json ./sbom.spdx.json
# Copy hardcoded dynamic libraries
COPY --from=builder /out/libs-root /
# Copy dynamic libraries
COPY --from=builder /out/libs /libs
# Tell Linux where to find our libraries
ENV LD_LIBRARY_PATH=/libs
# Copy dynamic libraries to root
COPY --from=builder /out/libs /
CMD ["/app"]