bazel: use remote caching (#1456)

* bazel: add configuration for remote caching
* ci: enable bazel remote caching for building binaries
* ci: use bazel directly when building go binaries
* ci: enable cache for most build steps
* dev-docs: document remote caching
This commit is contained in:
Malte Poll 2023-03-20 16:05:08 +01:00 committed by GitHub
parent 4f37fe38f9
commit c3c0940adb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 263 additions and 46 deletions

View File

@ -38,3 +38,12 @@ build:linux_arm64_static --platforms @zig_sdk//libc_aware/platform:linux_arm64_m
# bazel config to explicitly disable stamping (hide version information at build time)
build:nostamp --nostamp --workspace_status_command=
# bazel config to use remote cache
build:remote_cache --bes_results_url=https://app.buildbuddy.io/invocation/
build:remote_cache --bes_backend=grpcs://remote.buildbuddy.io
build:remote_cache --remote_cache=grpcs://remote.buildbuddy.io
build:remote_cache --remote_timeout=3600
build:remote_cache --experimental_remote_build_event_upload=minimal
build:remote_cache --experimental_remote_cache_compression
build:remote_cache_readonly --noremote_upload_local_results # Uploads logs & artifacts without writing to cache

View File

@ -4,7 +4,7 @@ description: Build the Constellation bootstrapper binary
inputs:
outputPath:
description: "Output path of the binary"
default: "./bootstrapper"
default: "./build/bootstrapper"
required: true
# Linux runner only (Docker required)
@ -13,10 +13,16 @@ runs:
steps:
- name: Build the bootstrapper
shell: bash
env:
OUTPUT_PATH: ${{ inputs.outputPath }}
run: |
echo "::group::Build the bootstrapper"
mkdir -p build && cd build
cmake -DBAZEL:STRING=bazelisk ..
make bootstrapper
mv -n bootstrapper "${{ inputs.outputPath }}"
mkdir -p "$(dirname "${OUTPUT_PATH}")"
label=//bootstrapper/cmd/bootstrapper:bootstrapper_linux_amd64
bazel build "${label}"
repository_root=$(git rev-parse --show-toplevel)
out_rel=$(bazel cquery --output=files "${label}")
out_loc="$(realpath "${repository_root}/${out_rel}")"
cp "${out_loc}" "${OUTPUT_PATH}"
chmod +w "${OUTPUT_PATH}"
echo "::endgroup::"

View File

@ -10,15 +10,28 @@ inputs:
description: "Build CLI for this architecture. [amd64, arm64]"
required: true
default: "amd64"
outputPath:
description: "Output path of the binary"
default: "./build/cdbg"
required: false
runs:
using: "composite"
steps:
- name: Build cdbg
shell: bash
env:
GOOS: ${{ inputs.targetOS }}
GOARCH: ${{ inputs.targetArch }}
OUTPUT_PATH: ${{ inputs.outputPath }}
run: |
echo "::group::Build cdbg"
mkdir -p build && cd build
cmake ..
GOOS=${{ inputs.targetOS }} GOARCH=${{ inputs.targetArch }} make cdbg
mkdir -p "$(dirname "${OUTPUT_PATH}")"
label="//debugd/cmd/cdbg:cdbg_${GOOS}_${GOARCH}"
bazel build "${label}"
repository_root=$(git rev-parse --show-toplevel)
out_rel=$(bazel cquery --output=files "${label}")
out_loc="$(realpath "${repository_root}/${out_rel}")"
cp "${out_loc}" "${OUTPUT_PATH}"
chmod +w "${OUTPUT_PATH}"
echo "::endgroup::"

View File

@ -1,6 +1,6 @@
name: Build CLI
description: |
Runs cmake and cli make target in build folder. Optionally, Sigstore tools
Builds CLI. Optionally, Sigstore tools
are used to sign CLI when inputs are provided. A draft release is published
when run on v* tag.
inputs:
@ -28,6 +28,9 @@ inputs:
description: "Password for Cosign private key"
required: false
default: ""
outputPath:
description: "Output path of the binary"
required: false
runs:
using: "composite"
steps:
@ -39,20 +42,28 @@ runs:
- name: Build CLI
shell: bash
env:
GOOS: ${{ inputs.targetOS }}
GOARCH: ${{ inputs.targetArch }}
OUTPUT_PATH: ${{ inputs.outputPath || format('./build/constellation-{0}-{1}', inputs.targetOS, inputs.targetArch) }}
run: |
echo "::group::Build CLI"
mkdir -p build
cd build
mkdir -p "$(dirname "${OUTPUT_PATH}")"
if [ ${{ inputs.enterpriseCLI }} == 'true' ]
then
cmake -DBAZEL:STRING=bazelisk -DCLI_BUILD_TAGS:STRING=enterprise ..
cli_variant=enterprise
else
cmake -DBAZEL:STRING=bazelisk ..
cli_variant=oss
fi
GOOS=${{ inputs.targetOS }} GOARCH=${{ inputs.targetArch }} make cli
cp constellation constellation-${{ inputs.targetOS }}-${{ inputs.targetArch }}
echo "$(pwd)" >> $GITHUB_PATH
export PATH="$PATH:$(pwd)"
label="//cli:cli_${cli_variant}_${GOOS}_${GOARCH}"
bazel build "${label}"
repository_root=$(git rev-parse --show-toplevel)
out_rel=$(bazel cquery --output=files "${label}")
out_loc="$(realpath "${repository_root}/${out_rel}")"
cp "${out_loc}" "${OUTPUT_PATH}"
chmod +w "${OUTPUT_PATH}"
echo "$(dirname "${OUTPUT_PATH}")" >> $GITHUB_PATH
export PATH="$PATH:$(dirname "${OUTPUT_PATH}")"
echo "::endgroup::"
# TODO: Replace with https://github.com/sigstore/sigstore-installer/tree/initial
@ -80,15 +91,15 @@ runs:
COSIGN_PUBLIC_KEY: ${{ inputs.cosignPublicKey }}
COSIGN_PRIVATE_KEY: ${{ inputs.cosignPrivateKey }}
COSIGN_PASSWORD: ${{ inputs.cosignPassword }}
OUTPUT_PATH: ${{ inputs.outputPath || format('./build/constellation-{0}-{1}', inputs.targetOS, inputs.targetArch) }}
run: |
SIGN_TARGET=constellation-${{ inputs.targetOS }}-${{ inputs.targetArch }}
echo "$COSIGN_PUBLIC_KEY" > cosign.pub
# Enabling experimental mode also publishes signature to Rekor
COSIGN_EXPERIMENTAL=1 cosign sign-blob --key env://COSIGN_PRIVATE_KEY ${SIGN_TARGET} > ${SIGN_TARGET}.sig
COSIGN_EXPERIMENTAL=1 cosign sign-blob --key env://COSIGN_PRIVATE_KEY "${OUTPUT_PATH}" > "${OUTPUT_PATH}.sig"
# Verify - As documentation & check
# Local Signature (input: artifact, key, signature)
cosign verify-blob --key cosign.pub --signature ${SIGN_TARGET}.sig ${SIGN_TARGET}
cosign verify-blob --key cosign.pub --signature "${OUTPUT_PATH}.sig" "${OUTPUT_PATH}"
# Transparency Log Signature (input: artifact, key)
uuid=$(rekor-cli search --artifact ${SIGN_TARGET} | tail -n 1)
uuid=$(rekor-cli search --artifact "${OUTPUT_PATH}" | tail -n 1)
sig=$(rekor-cli get --uuid=$uuid --format=json | jq -r .Body.HashedRekordObj.signature.content)
cosign verify-blob --key cosign.pub --signature <(echo $sig) ${SIGN_TARGET}
cosign verify-blob --key cosign.pub --signature <(echo $sig) "${OUTPUT_PATH}"

View File

@ -4,7 +4,7 @@ description: Build the Constellation debugd binary
inputs:
outputPath:
description: "Output path of the binary"
default: "./debugd"
default: "./build/debugd"
required: true
# Linux runner only (homedir trick does not work on macOS, required for private runner)
@ -13,14 +13,16 @@ runs:
steps:
- name: Build debugd
shell: bash
env:
OUTPUT_PATH: ${{ inputs.outputPath }}
run: |
echo "::group::Build debugd"
homedir="$(getent passwd $(id -u) | cut -d ":" -f 6)"
mkdir -p build && cd build
cmake -DBAZEL:STRING=bazelisk ..
export GOCACHE=${homedir}/.cache/go-build
export GOPATH=${homedir}/go
export GOMODCACHE=${homedir}/.cache/go-mod
make debugd
mv -n debugd "${{ inputs.outputPath }}"
mkdir -p "$(dirname "${OUTPUT_PATH}")"
label=//debugd/cmd/debugd:debugd_linux_amd64
bazel build "${label}"
repository_root=$(git rev-parse --show-toplevel)
out_rel=$(bazel cquery --output=files "${label}")
out_loc="$(realpath "${repository_root}/${out_rel}")"
cp "${out_loc}" "${OUTPUT_PATH}"
chmod +w "${OUTPUT_PATH}"
echo "::endgroup::"

View File

@ -4,7 +4,7 @@ description: Build the Constellation disk-mapper binary
inputs:
outputPath:
description: "Output path of the binary"
default: "./disk-mapper"
default: "./build/disk-mapper"
required: true
# Linux runner only (Docker required)
@ -13,10 +13,16 @@ runs:
steps:
- name: Build the disk-mapper
shell: bash
env:
OUTPUT_PATH: ${{ inputs.outputPath }}
run: |
echo "::group::Build the disk-mapper"
mkdir -p build && cd build
cmake -DBAZEL:STRING=bazelisk ..
make disk-mapper
mv -n disk-mapper "${{ inputs.outputPath }}"
mkdir -p "$(dirname "${OUTPUT_PATH}")"
label="//disk-mapper/cmd:disk-mapper_linux_amd64"
bazel build "${label}"
repository_root=$(git rev-parse --show-toplevel)
out_rel=$(bazel cquery --output=files "${label}")
out_loc="$(realpath "${repository_root}/${out_rel}")"
cp "${out_loc}" "${OUTPUT_PATH}"
chmod +w "${OUTPUT_PATH}"
echo "::endgroup::"

View File

@ -4,7 +4,7 @@ description: Build the Constellation measurement-reader binary
inputs:
outputPath:
description: "Output path of the binary"
default: "./measurement-reader"
default: "./build/measurement-reader"
required: true
# Linux runner only (Docker required)
@ -13,10 +13,16 @@ runs:
steps:
- name: Build the measurement-reader
shell: bash
env:
OUTPUT_PATH: ${{ inputs.outputPath }}
run: |
echo "::group::Build the measurement-reader"
mkdir -p build && cd build
cmake ..
make measurement-reader
mv -n measurement-reader "${{ inputs.outputPath }}"
mkdir -p "$(dirname "${OUTPUT_PATH}")"
label="//measurement-reader/cmd:measurement-reader_linux_amd64"
bazel build "${label}"
repository_root=$(git rev-parse --show-toplevel)
out_rel=$(bazel cquery --output=files "${label}")
out_loc="$(realpath "${repository_root}/${out_rel}")"
cp "${out_loc}" "${OUTPUT_PATH}"
chmod +w "${OUTPUT_PATH}"
echo "::endgroup::"

View File

@ -4,7 +4,7 @@ description: Build the Constellation upgrade-agent binary
inputs:
outputPath:
description: "Output path of the binary"
default: "./upgrade-agent"
default: "./build/upgrade-agent"
required: true
# Linux runner only (Docker required)
@ -13,10 +13,16 @@ runs:
steps:
- name: Build the upgrade-agent
shell: bash
env:
OUTPUT_PATH: ${{ inputs.outputPath }}
run: |
echo "::group::Build the upgrade-agent"
mkdir -p build && cd build
cmake -DBAZEL:STRING=bazelisk ..
make upgrade-agent
mv -n upgrade-agent "${{ inputs.outputPath }}"
mkdir -p "$(dirname "${OUTPUT_PATH}")"
label="//upgrade-agent/cmd:upgrade_agent_linux_amd64"
bazel build "${label}"
repository_root=$(git rev-parse --show-toplevel)
out_rel=$(bazel cquery --output=files "${label}")
out_loc="$(realpath "${repository_root}/${out_rel}")"
cp "${out_loc}" "${OUTPUT_PATH}"
chmod +w "${OUTPUT_PATH}"
echo "::endgroup::"

View File

@ -11,6 +11,9 @@ inputs:
azureTenantID:
description: "Azure tenant to use for login with OIDC"
required: true
buildBuddyApiKey:
description: "BuildBuddy API key for caching Bazel artifacts"
required: true
runs:
using: "composite"
@ -25,10 +28,17 @@ runs:
with:
go-version: "1.20.2"
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "true"
buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }}
- name: Build Constellation CLI
uses: ./.github/actions/build_cli
with:
enterpriseCLI: true
outputPath: build/constellation
- name: MiniConstellation E2E
shell: bash

View File

@ -71,6 +71,9 @@ inputs:
sonobuoyTestSuiteCmd:
description: "The sonobuoy test suite to run."
required: false
buildBuddyApiKey:
description: "BuildBuddy API key for caching Bazel artifacts"
required: true
outputs:
kubeconfig:
@ -104,12 +107,19 @@ runs:
echo "hostOS=$(go env GOOS)" >> $GITHUB_OUTPUT
echo "hostArch=$(go env GOARCH)" >> $GITHUB_OUTPUT
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "true"
buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }}
- name: Build CLI
uses: ./.github/actions/build_cli
with:
targetOS: ${{ steps.determine-build-target.outputs.hostOS }}
targetArch: ${{ steps.determine-build-target.outputs.hostArch }}
enterpriseCLI: ${{ inputs.keepMeasurements }}
outputPath: "build/constellation"
- name: Build the bootstrapper
id: build-bootstrapper

55
.github/actions/setup_bazel/action.yml vendored Normal file
View File

@ -0,0 +1,55 @@
name: Setup bazel
description: Setup Bazel for CI builds and tests
inputs:
useCache:
description: "Cache Bazel artifacts. Use 'true' to enable with rw, 'readonly' to download, and 'false' to disable."
default: "false"
required: true
buildBuddyApiKey:
description: "BuildBuddy API key for caching Bazel artifacts"
required: false
runs:
using: "composite"
steps:
- name: Check inputs
shell: bash
run: |
echo "::group::Check inputs"
if [[ "${{ inputs.useCache }}" != "true" && "${{ inputs.useCache }}" != "readonly" && "${{ inputs.useCache }}" != "false" ]]; then
echo "Invalid value for 'useCache' input: '${{ inputs.useCache }}'. Must be 'true', 'readonly', or 'false'."
exit 1
fi
if [[ "${{ inputs.useCache }}" == "true" || "${{ inputs.useCache }}" == "readonly" ]] && [[ -z "${{ inputs.buildBuddyApiKey }}" ]]; then
echo "BuildBuddy API key is required when cache is enabled."
exit 1
fi
echo "::endgroup::"
- name: Configure Bazel
shell: bash
if: inputs.useCache == 'true' || inputs.useCache == 'readonly'
env:
BUILDBUDDY_ORG_API_KEY: ${{ inputs.buildBuddyApiKey }}
run: |
echo "::group::Configure Bazel"
cat <<EOF >> ~/.bazelrc
build --bes_results_url=https://app.buildbuddy.io/invocation/
build --bes_backend=grpcs://remote.buildbuddy.io
build --remote_cache=grpcs://remote.buildbuddy.io
build --remote_timeout=3600
build --experimental_remote_build_event_upload=minimal
build --experimental_remote_cache_compression
build --remote_header=x-buildbuddy-api-key=${BUILDBUDDY_ORG_API_KEY}
EOF
echo "::endgroup::"
- name: Configure Bazel (readonly)
shell: bash
if: inputs.useCache == 'readonly'
run: |
echo "::group::Configure Bazel (readonly)"
echo "build --remote_upload_local_results=false" >> ~/.bazelrc
echo "::endgroup::"
- name: Check bazel version
shell: bash
run: bazel version

View File

@ -26,6 +26,12 @@ jobs:
with:
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "true"
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
- name: Build the bootstrapper
uses: ./.github/actions/build_bootstrapper
@ -37,6 +43,12 @@ jobs:
with:
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "true"
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
- name: Build debugd
uses: ./.github/actions/build_debugd
@ -48,17 +60,25 @@ jobs:
with:
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "true"
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
- name: Build cdbg (Linux, amd64)
uses: ./.github/actions/build_cdbg
with:
targetOS: "linux"
targetArch: "amd64"
outputPath: ./build/cdbg_linux_amd64
- name: Build cdbg (Linux, arm64)
uses: ./.github/actions/build_cdbg
with:
targetOS: "linux"
targetArch: "arm64"
outputPath: ./build/cdbg_linux_arm64
build-cdbg-macos:
runs-on: ubuntu-22.04
@ -68,17 +88,25 @@ jobs:
with:
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "true"
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
- name: Build cdbg (macOS, amd64)
uses: ./.github/actions/build_cdbg
with:
targetOS: "darwin"
targetArch: "amd64"
outputPath: ./build/cdbg_darwin_amd64
- name: Build cdbg (macOS, arm64)
uses: ./.github/actions/build_cdbg
with:
targetOS: "darwin"
targetArch: "arm64"
outputPath: ./build/cdbg_darwin_arm64
build-disk-mapper:
runs-on: ubuntu-22.04
@ -88,6 +116,12 @@ jobs:
with:
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "true"
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
- name: Build disk-mapper
uses: ./.github/actions/build_disk_mapper
@ -99,6 +133,12 @@ jobs:
with:
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "true"
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
- name: Build measurement-reader
uses: ./.github/actions/build_measurement_reader
@ -110,6 +150,12 @@ jobs:
with:
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "true"
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
- name: Build CLI (Linux, amd64)
uses: ./.github/actions/build_cli
with:
@ -130,6 +176,12 @@ jobs:
with:
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "true"
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
- name: Build CLI (macOS, amd64)
uses: ./.github/actions/build_cli
with:

View File

@ -68,6 +68,11 @@ jobs:
go-version: "1.20.2"
cache: true
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "false"
- name: Build bootstrapper
if: inputs.stream != 'debug'
uses: ./.github/actions/build_bootstrapper

View File

@ -41,3 +41,4 @@ jobs:
azureClientID: ${{ secrets.AZURE_E2E_MINI_CLIENT_ID }}
azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
azureTenantID: ${{ secrets.AZURE_TENANT_ID }}
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}

View File

@ -96,6 +96,7 @@ jobs:
gcp_service_account: "constellation-e2e@constellation-331613.iam.gserviceaccount.com"
gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
test: "sonobuoy full"
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
- name: Always terminate cluster
if: always()

View File

@ -258,6 +258,7 @@ jobs:
azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
osImage: ${{ needs.find-latest-image.outputs.image }}
isDebugImage: ${{ needs.find-latest-image.outputs.isDebugImage }}
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
- name: Always terminate cluster
if: always()

View File

@ -148,6 +148,7 @@ jobs:
gcp_service_account: "constellation-e2e@constellation-331613.iam.gserviceaccount.com"
gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
test: ${{ matrix.test }}
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
- name: Always terminate cluster
if: always()

View File

@ -29,6 +29,11 @@ jobs:
with:
ref: ${{ inputs.ref || github.head_ref }}
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "false"
- name: Build CLI
uses: ./.github/actions/build_cli
with:
@ -107,6 +112,11 @@ jobs:
- name: Download Syft & Grype
uses: ./.github/actions/install_syft_grype
- name: Setup bazel
uses: ./.github/actions/setup_bazel
with:
useCache: "false"
# Build one CLI since Syft's go-module catalog will default to binary parsing.
# Binary parsing has the advantage that it will not include other dependencies from our repo not included in the CLI.
# This seems to work fine for one OS & one arch as long as we don't have OS specific imports.

View File

@ -50,6 +50,17 @@ bazel build //cli:cli_oss_darwin_amd64 # cross compile CLI for mac amd64
bazel build //cli:cli_oss_darwin_arm64 # cross compile CLI for mac arm64
```
## Remote caching and execution
We use BuildBuddy for remote caching (and maybe remote execution in the future). To use it, you need to join the BuildBuddy organization and get an API key. Then, you can write it to `~/.bazelrc`:
```
build --remote_header=x-buildbuddy-api-key=<redacted>
```
To use the remote cache, build the project with `bazel build --config remote_cache //path/to:target`.
You can also copy the `remote_cache` config from `.bazelrc` to your `~/.bazelrc` and remove the `remote_cache` prefix to make it the default.
# Test
You can run all integration and unitttests like this:

View File

@ -22,6 +22,7 @@ timestamp() {
"${REPOSITORY_ROOT}/tools/pseudo-version" -print-timestamp -timestamp-format '2006-01-02T15:04:05Z07:00'
}
echo "REPO_URL https://github.com/edgelesssys/constellation.git"
echo "STABLE_STAMP_COMMIT $(git rev-parse HEAD)"
echo "STABLE_STAMP_STATE $(git diff-index --quiet HEAD -- && echo "clean" || echo "dirty")"
echo "STABLE_STAMP_VERSION $(pseudo_version)"