Merge branch 'earthly-caching-2' into 'main'

Refactored Earthfile to improve caching See merge request veilid/veilid!232
2025-04-14 21:13:03 -04:00 · 2023-11-03 01:17:14 +00:00 · 2023-11-03 01:17:14 +00:00 · 502d08eda3
commit 502d08eda3
parent 5691e5a6ac d38f3cb42f
9 changed files with 153 additions and 79 deletions
--- a/.gitignore
+++ b/.gitignore
@ -236,4 +236,7 @@ veilid-python/demo/.demokeys
 .vscode/

 ## Ignore emacs backup files
-~*
+~*
+
+# Earthly temporary build output
+.tmp-earthly-out/
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -5,29 +5,53 @@ variables:
  GIT_SUBMODULE_STRATEGY: normal

 stages:
+  - prepare
  - test
  - build_packages
  - release
  - distribute

-unit_test_linux:
-  stage: test
+# base earthly setup for jobs
+.base:
+  tags: [ saas-linux-medium-amd64 ]
  image: docker
  services:
    - docker:dind
+
+.earthly: &earthly_setup
+  - apk update && apk add git
+  - wget https://github.com/earthly/earthly/releases/download/v0.7.15/earthly-linux-amd64 -O /usr/local/bin/earthly
+  - chmod +x /usr/local/bin/earthly
+  - earthly bootstrap
+  - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+  - test "$CI_PROJECT_PATH" != "veilid/veilid" && project_args="--CI_PROJECT_PATH=$CI_PROJECT_PATH"
+
+# Create the build container if:
+#  - no container in the registry
+#  - run as part of a schedule
+#  - run manually from the pipelines web page
+build_cache:
+  extends: .base
+  stage: prepare
+  script:
+    - if ! docker manifest inspect registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest > /dev/null || [[ "$CI_PIPELINE_SOURCE" == "schedule" ]] || [[ "$CI_PIPELINE_SOURCE" == "web" ]] ; then
+    -   *earthly_setup
+    -   earthly --use-inline-cache --save-inline-cache --strict --push -P +build-linux-cache $project_args
+    - else
+    -   echo "No need to rebuild"
+    - fi
+
+# Runs a basic unit test build, this task will use the `build-cache:latest` as set up in the projects Container Registry
+test_build:
+  extends: .base
+  stage: test
  only:
    - main
+    - pushes
    - merge_requests
-  before_script:
-    - apk update && apk add git
-    - wget https://github.com/earthly/earthly/releases/download/v0.6.2/earthly-linux-amd64 -O /usr/local/bin/earthly
-    - chmod +x /usr/local/bin/earthly
-    - earthly bootstrap
-    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
  script:
-    - if [ "$CI_COMMIT_BRANCH" == "main" ]; then cache_args="--save-inline-cache --push"; fi
-    - if [ "$CI_PROJECT_PATH" != "veilid/veilid" ]; then project_args="--CI_PROJECT_PATH=$CI_PROJECT_PATH"; fi
-    - earthly --use-inline-cache $cache_args --ci +unit-tests-linux $project_args
+    - *earthly_setup
+    - earthly --use-inline-cache +unit-tests-linux --BASE=container $project_args
  resource_group: test
  when: manual

--- a/174
+++ b/174
@ -1,10 +1,27 @@
-VERSION 0.6
+VERSION 0.7
+
+########################################################################################################################
+## ARGUMENTS
+##
+## CI_PROJECT_PATH - used so that forks can refer to themselves, e.g. to use the fork's own registry cache in the
+## `+build-linux-cache` target, and defaulting to `veilid/veilid` if not specified
+##
+## BASE - tells the build whether it should run in the default mode which runs the complete build, or run by starting
+## with the remote `container` value which uses `build-cache:latest` as set up in the projects Container Registry
+##
+########################################################################################################################

 # Start with older Ubuntu to ensure GLIBC symbol versioning support for older linux
 # Ensure we are using an amd64 platform because some of these targets use cross-platform tooling
 FROM ubuntu:18.04
+ENV ZIG_VERSION=0.11.0-dev.3978+711b4e93e
+ENV RUSTUP_HOME=/usr/local/rustup
+ENV CARGO_HOME=/usr/local/cargo
+ENV PATH=$PATH:/usr/local/cargo/bin:/usr/local/zig
+ENV LD_LIBRARY_PATH=/usr/local/lib
+WORKDIR /veilid

-# Install build prerequisites
+# Install build prerequisites & setup required directories
 deps-base:
    RUN apt-get -y update
    RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip libncursesw5-dev libncurses5-dev
@ -14,25 +31,27 @@ deps-capnp:
    FROM +deps-base
    COPY scripts/earthly/install_capnproto.sh /
    RUN /bin/bash /install_capnproto.sh 1; rm /install_capnproto.sh
+    SAVE ARTIFACT /usr/local/bin/capnp* bin/
+    SAVE ARTIFACT /usr/local/lib/lib* lib/
+    SAVE ARTIFACT /usr/local/lib/pkgconfig lib/

 # Install protoc
 deps-protoc:
-    FROM +deps-capnp
+    FROM +deps-base
    COPY scripts/earthly/install_protoc.sh /
    RUN /bin/bash /install_protoc.sh; rm /install_protoc.sh
+    SAVE ARTIFACT /usr/local/bin/protoc bin/protoc

 # Install Rust
 deps-rust:
-    FROM +deps-protoc
-    ENV RUSTUP_HOME=/usr/local/rustup
-    ENV CARGO_HOME=/usr/local/cargo
-    ENV PATH=/usr/local/cargo/bin:$PATH
-    RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal 
+    FROM +deps-base
+    RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal
    RUN chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
        rustup --version; \
        cargo --version; \
        rustc --version;
-    # ARM64 Linux
+    # Linux
+    RUN rustup target add x86_64-unknown-linux-gnu
    RUN rustup target add aarch64-unknown-linux-gnu
    # Android
    RUN rustup target add aarch64-linux-android
@ -41,20 +60,31 @@ deps-rust:
    RUN rustup target add x86_64-linux-android
    # WASM
    RUN rustup target add wasm32-unknown-unknown
-
-# Install Linux cross-platform tooling
-deps-cross:
-    FROM +deps-rust
-    RUN curl https://ziglang.org/builds/zig-linux-x86_64-0.11.0-dev.3978+711b4e93e.tar.xz | tar -C /usr/local -xJf -
-    RUN mv /usr/local/zig-linux-x86_64-0.11.0-dev.3978+711b4e93e /usr/local/zig
-    ENV PATH=$PATH:/usr/local/zig
+    # Caching tool
+    RUN cargo install cargo-chef
+    # Install Linux cross-platform tooling
+    RUN curl -O https://ziglang.org/builds/zig-linux-x86_64-$ZIG_VERSION.tar.xz
+    RUN tar -C /usr/local -xJf zig-linux-x86_64-$ZIG_VERSION.tar.xz
+    RUN mv /usr/local/zig-linux-x86_64-$ZIG_VERSION /usr/local/zig
    RUN cargo install cargo-zigbuild
-    RUN rustup target add x86_64-unknown-linux-gnu
-    RUN rustup target add aarch64-unknown-linux-gnu
+    SAVE ARTIFACT $RUSTUP_HOME rustup
+    SAVE ARTIFACT $CARGO_HOME cargo
+    SAVE ARTIFACT /usr/local/cargo/bin/cargo-zigbuild
+    SAVE ARTIFACT /usr/local/zig

 # Install android tooling
 deps-android:
-    FROM +deps-cross
+    FROM +deps-base
+    BUILD +deps-protoc
+    COPY +deps-protoc/bin/* /usr/local/bin/
+    BUILD +deps-capnp
+    COPY +deps-capnp/bin/* /usr/local/bin/
+    COPY +deps-capnp/lib/* /usr/local/lib/
+    BUILD +deps-rust
+    COPY +deps-rust/cargo /usr/local/cargo
+    COPY +deps-rust/rustup /usr/local/rustup
+    COPY +deps-rust/cargo-zigbuild /usr/local/cargo/bin/cargo-zigbuild
+    COPY +deps-rust/zig /usr/local/zig
    RUN apt-get install -y openjdk-9-jdk-headless
    RUN mkdir /Android; mkdir /Android/Sdk
    RUN curl -o /Android/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-9123335_latest.zip
@ -62,18 +92,51 @@ deps-android:
    RUN yes | /Android/cmdline-tools/bin/sdkmanager --sdk_root=/Android/Sdk build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33 cmdline-tools\;latest
    RUN rm -rf /Android/cmdline-tools
    RUN apt-get clean
-    
+
 # Just linux build not android
 deps-linux:
-    FROM +deps-cross
-    RUN apt-get clean
+    FROM +deps-base
+    BUILD +deps-protoc
+    COPY +deps-protoc/bin/* /usr/local/bin/
+    BUILD +deps-capnp
+    COPY +deps-capnp/bin/* /usr/local/bin/
+    COPY +deps-capnp/lib/* /usr/local/lib/
+    BUILD +deps-rust
+    COPY +deps-rust/cargo /usr/local/cargo
+    COPY +deps-rust/rustup /usr/local/rustup
+    COPY +deps-rust/cargo-zigbuild /usr/local/cargo/bin/cargo-zigbuild
+    COPY +deps-rust/zig /usr/local/zig

-# Code + Linux deps
-code-linux:
+build-linux-cache:
    FROM +deps-linux
+    RUN mkdir veilid-cli veilid-core veilid-server veilid-tools veilid-wasm veilid-flutter veilid-flutter/rust
+    COPY --dir .cargo scripts Cargo.lock Cargo.toml .
+    COPY veilid-cli/Cargo.toml veilid-cli
+    COPY veilid-core/Cargo.toml veilid-core
+    COPY veilid-server/Cargo.toml veilid-server
+    COPY veilid-tools/Cargo.toml veilid-tools
+    COPY veilid-flutter/rust/Cargo.lock veilid-flutter/rust/Cargo.toml veilid-flutter/rust
+    COPY veilid-wasm/Cargo.toml veilid-wasm
+    RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> .cargo/config.toml
+    RUN cargo chef prepare --recipe-path recipe.json
+    RUN cargo chef cook --recipe-path recipe.json
+    RUN echo $PROJECT_PATH
+    SAVE ARTIFACT target
+    ARG CI_PROJECT_PATH=veilid/veilid
+    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest
+
+code-linux:
+    # This target will either use the full earthly cache of local use (+build-linux-cache), or will use a containerized
+    # version of the +build-linux-cache from the registry
+    ARG BASE=local
+    IF [ "$BASE" = "local" ]
+        FROM +build-linux-cache
+    ELSE
+        ARG CI_PROJECT_PATH=veilid/veilid
+        FROM registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest
+        # FROM registry.gitlab.com/veilid/build-cache:latest
+    END
    COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid
-    RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml
-    WORKDIR /veilid

 # Code + Linux + Android deps
 code-android:
@ -81,89 +144,68 @@ code-android:
    COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid
    RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml
    RUN cat /veilid/scripts/earthly/cargo-android/config.toml >> /veilid/.cargo/config.toml
-    WORKDIR /veilid

 # Clippy only
 clippy:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    RUN cargo clippy
-    SAVE IMAGE --push registry.gitl ab.com/$CI_PROJECT_PATH/clippy:latest

 # Build
 build-release:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    RUN cargo build --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE ARTIFACT ./target/release AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-release:latest
+    SAVE ARTIFACT ./target/release AS LOCAL ./target/release

 build:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    RUN cargo build -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE ARTIFACT ./target/debug AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build:latest
+    SAVE ARTIFACT ./target/debug AS LOCAL ./target/debug

 build-linux-amd64:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
-    RUN cargo build --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
+    RUN cargo zigbuild --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
    SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-amd64:latest

 build-linux-amd64-debug:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
-    RUN cargo build --target x86_64-unknown-linux-gnu -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
+    RUN cargo zigbuild --target x86_64-unknown-linux-gnu -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
    SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-amd64-debug:latest

 build-linux-arm64:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    RUN cargo zigbuild --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
    SAVE ARTIFACT ./target/aarch64-unknown-linux-gnu AS LOCAL ./target/artifacts/aarch64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-arm64:latest

-# build-android:
-#     FROM +code-android
-#     ARG CI_PROJECT_PATH=veilid/veilid
-#     WORKDIR /veilid/veilid-core
-#     ENV PATH=$PATH:/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/
-#     RUN cargo build --target aarch64-linux-android --release
-#     RUN cargo build --target armv7-linux-androideabi --release
-#     RUN cargo build --target i686-linux-android --release
-#     RUN cargo build --target x86_64-linux-android --release
-#     WORKDIR /veilid
-#     SAVE ARTIFACT ./target/aarch64-linux-android AS LOCAL ./target/artifacts/aarch64-linux-android
-#     SAVE ARTIFACT ./target/armv7-linux-androideabi AS LOCAL ./target/artifacts/armv7-linux-androideabi
-#     SAVE ARTIFACT ./target/i686-linux-android AS LOCAL ./target/artifacts/i686-linux-android
-#     SAVE ARTIFACT ./target/x86_64-linux-android AS LOCAL ./target/artifacts/x86_64-linux-android
-#     SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-android:latest
+build-android:
+    FROM +code-android
+    WORKDIR /veilid/veilid-core
+    ENV PATH=$PATH:/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/
+    RUN cargo build --target aarch64-linux-android --release
+    RUN cargo build --target armv7-linux-androideabi --release
+    RUN cargo build --target i686-linux-android --release
+    RUN cargo build --target x86_64-linux-android --release
+    WORKDIR /veilid
+    SAVE ARTIFACT ./target/aarch64-linux-android AS LOCAL ./target/artifacts/aarch64-linux-android
+    SAVE ARTIFACT ./target/armv7-linux-androideabi AS LOCAL ./target/artifacts/armv7-linux-androideabi
+    SAVE ARTIFACT ./target/i686-linux-android AS LOCAL ./target/artifacts/i686-linux-android
+    SAVE ARTIFACT ./target/x86_64-linux-android AS LOCAL ./target/artifacts/x86_64-linux-android

 # Unit tests
 unit-tests-linux:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    ENV RUST_BACKTRACE=1
-    RUN cargo test --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux:latest
+    RUN cargo test -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core

 # TODO: Change t0 cross so that they work on any platform
 unit-tests-linux-amd64:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    ENV RUST_BACKTRACE=1
    RUN cargo test --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux-amd64:latest

 unit-tests-linux-arm64:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    ENV RUST_BACKTRACE=1
    RUN cargo test --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux-arm64:latest

 # Package 
 package-linux-amd64-deb:
@ -232,8 +274,6 @@ package-linux-arm64-rpm:
    # save artifacts
    SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/

-
-
 package-linux-amd64:
    BUILD +package-linux-amd64-deb
    BUILD +package-linux-amd64-rpm
--- a/veilid-cli/Cargo.toml
+++ b/veilid-cli/Cargo.toml
@ -6,6 +6,7 @@ version = "0.2.4"
 authors = ["Veilid Team <contact@veilid.com>"]
 edition = "2021"
 license = "MPL-2.0"
+resolver = "2"

 [[bin]]
 name = "veilid-cli"
--- a/veilid-core/Cargo.toml
+++ b/veilid-core/Cargo.toml
@ -8,9 +8,11 @@ authors = ["Veilid Team <contact@veilid.com>"]
 edition = "2021"
 build = "build.rs"
 license = "MPL-2.0"
+resolver = "2"

 [lib]
 crate-type = ["cdylib", "staticlib", "rlib"]
+path = "src/lib.rs"

 [features]

--- a/veilid-flutter/rust/Cargo.toml
+++ b/veilid-flutter/rust/Cargo.toml
@ -9,6 +9,7 @@ edition = "2021"

 [lib]
 crate-type = ["cdylib", "staticlib", "rlib"]
+path = "src/lib.rs"

 [features]
 default = ["rt-tokio", "veilid-core/default"]
--- a/veilid-server/Cargo.toml
+++ b/veilid-server/Cargo.toml
@ -7,6 +7,7 @@ description = "Veilid Server"
 authors = ["Veilid Team <contact@veilid.com>"]
 license = "MPL-2.0"
 edition = "2021"
+resolver = "2"

 [[bin]]
 name = "veilid-server"
--- a/veilid-tools/Cargo.toml
+++ b/veilid-tools/Cargo.toml
@ -11,6 +11,7 @@ edition = "2021"
 [lib]
 # staticlib for iOS tests, cydlib for android tests, rlib for everything else
 crate-type = ["cdylib", "staticlib", "rlib"]
+path = "src/lib.rs"

 [features]
 default = ["rt-tokio"]
--- a/veilid-wasm/Cargo.toml
+++ b/veilid-wasm/Cargo.toml
@ -9,6 +9,7 @@ edition = "2021"

 [lib]
 crate-type = ["cdylib", "rlib"]
+path = "src/lib.rs"

 [features]
 default = ["veilid-core/default-wasm"]