From c3c0940adb20121002a46632a4926a333e760080 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Mon, 20 Mar 2023 16:05:08 +0100 Subject: [PATCH] bazel: use remote caching (#1456) * bazel: add configuration for remote caching * ci: enable bazel remote caching for building binaries * ci: use bazel directly when building go binaries * ci: enable cache for most build steps * dev-docs: document remote caching --- .bazelrc | 9 +++ .github/actions/build_bootstrapper/action.yml | 16 ++++-- .github/actions/build_cdbg/action.yml | 19 ++++++- .github/actions/build_cli/action.yml | 39 ++++++++----- .github/actions/build_debugd/action.yml | 20 ++++--- .github/actions/build_disk_mapper/action.yml | 16 ++++-- .../build_measurement_reader/action.yml | 16 ++++-- .../actions/build_upgrade_agent/action.yml | 16 ++++-- .github/actions/e2e_mini/action.yml | 10 ++++ .github/actions/e2e_test/action.yml | 10 ++++ .github/actions/setup_bazel/action.yml | 55 +++++++++++++++++++ .github/workflows/build-binaries.yml | 52 ++++++++++++++++++ .github/workflows/build-os-image.yml | 5 ++ .github/workflows/e2e-mini.yml | 1 + .github/workflows/e2e-test-daily.yml | 1 + .github/workflows/e2e-test-manual.yml | 1 + .github/workflows/e2e-test-weekly.yml | 1 + .github/workflows/release-cli.yml | 10 ++++ dev-docs/workflows/build-test-run.md | 11 ++++ tools/workspace_status.sh | 1 + 20 files changed, 263 insertions(+), 46 deletions(-) create mode 100644 .github/actions/setup_bazel/action.yml diff --git a/.bazelrc b/.bazelrc index 6ec0b5b0b..cea4ba90a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -38,3 +38,12 @@ build:linux_arm64_static --platforms @zig_sdk//libc_aware/platform:linux_arm64_m # bazel config to explicitly disable stamping (hide version information at build time) build:nostamp --nostamp --workspace_status_command= + +# bazel config to use remote cache +build:remote_cache --bes_results_url=https://app.buildbuddy.io/invocation/ +build:remote_cache --bes_backend=grpcs://remote.buildbuddy.io +build:remote_cache --remote_cache=grpcs://remote.buildbuddy.io +build:remote_cache --remote_timeout=3600 +build:remote_cache --experimental_remote_build_event_upload=minimal +build:remote_cache --experimental_remote_cache_compression +build:remote_cache_readonly --noremote_upload_local_results # Uploads logs & artifacts without writing to cache diff --git a/.github/actions/build_bootstrapper/action.yml b/.github/actions/build_bootstrapper/action.yml index b38099c12..e4209604d 100644 --- a/.github/actions/build_bootstrapper/action.yml +++ b/.github/actions/build_bootstrapper/action.yml @@ -4,7 +4,7 @@ description: Build the Constellation bootstrapper binary inputs: outputPath: description: "Output path of the binary" - default: "./bootstrapper" + default: "./build/bootstrapper" required: true # Linux runner only (Docker required) @@ -13,10 +13,16 @@ runs: steps: - name: Build the bootstrapper shell: bash + env: + OUTPUT_PATH: ${{ inputs.outputPath }} run: | echo "::group::Build the bootstrapper" - mkdir -p build && cd build - cmake -DBAZEL:STRING=bazelisk .. - make bootstrapper - mv -n bootstrapper "${{ inputs.outputPath }}" + mkdir -p "$(dirname "${OUTPUT_PATH}")" + label=//bootstrapper/cmd/bootstrapper:bootstrapper_linux_amd64 + bazel build "${label}" + repository_root=$(git rev-parse --show-toplevel) + out_rel=$(bazel cquery --output=files "${label}") + out_loc="$(realpath "${repository_root}/${out_rel}")" + cp "${out_loc}" "${OUTPUT_PATH}" + chmod +w "${OUTPUT_PATH}" echo "::endgroup::" diff --git a/.github/actions/build_cdbg/action.yml b/.github/actions/build_cdbg/action.yml index c10836f92..385d31715 100644 --- a/.github/actions/build_cdbg/action.yml +++ b/.github/actions/build_cdbg/action.yml @@ -10,15 +10,28 @@ inputs: description: "Build CLI for this architecture. [amd64, arm64]" required: true default: "amd64" + outputPath: + description: "Output path of the binary" + default: "./build/cdbg" + required: false runs: using: "composite" steps: - name: Build cdbg shell: bash + env: + GOOS: ${{ inputs.targetOS }} + GOARCH: ${{ inputs.targetArch }} + OUTPUT_PATH: ${{ inputs.outputPath }} run: | echo "::group::Build cdbg" - mkdir -p build && cd build - cmake .. - GOOS=${{ inputs.targetOS }} GOARCH=${{ inputs.targetArch }} make cdbg + mkdir -p "$(dirname "${OUTPUT_PATH}")" + label="//debugd/cmd/cdbg:cdbg_${GOOS}_${GOARCH}" + bazel build "${label}" + repository_root=$(git rev-parse --show-toplevel) + out_rel=$(bazel cquery --output=files "${label}") + out_loc="$(realpath "${repository_root}/${out_rel}")" + cp "${out_loc}" "${OUTPUT_PATH}" + chmod +w "${OUTPUT_PATH}" echo "::endgroup::" diff --git a/.github/actions/build_cli/action.yml b/.github/actions/build_cli/action.yml index a92bfd1b4..08a234480 100644 --- a/.github/actions/build_cli/action.yml +++ b/.github/actions/build_cli/action.yml @@ -1,6 +1,6 @@ name: Build CLI description: | - Runs cmake and cli make target in build folder. Optionally, Sigstore tools + Builds CLI. Optionally, Sigstore tools are used to sign CLI when inputs are provided. A draft release is published when run on v* tag. inputs: @@ -28,6 +28,9 @@ inputs: description: "Password for Cosign private key" required: false default: "" + outputPath: + description: "Output path of the binary" + required: false runs: using: "composite" steps: @@ -39,20 +42,28 @@ runs: - name: Build CLI shell: bash + env: + GOOS: ${{ inputs.targetOS }} + GOARCH: ${{ inputs.targetArch }} + OUTPUT_PATH: ${{ inputs.outputPath || format('./build/constellation-{0}-{1}', inputs.targetOS, inputs.targetArch) }} run: | echo "::group::Build CLI" - mkdir -p build - cd build + mkdir -p "$(dirname "${OUTPUT_PATH}")" if [ ${{ inputs.enterpriseCLI }} == 'true' ] then - cmake -DBAZEL:STRING=bazelisk -DCLI_BUILD_TAGS:STRING=enterprise .. + cli_variant=enterprise else - cmake -DBAZEL:STRING=bazelisk .. + cli_variant=oss fi - GOOS=${{ inputs.targetOS }} GOARCH=${{ inputs.targetArch }} make cli - cp constellation constellation-${{ inputs.targetOS }}-${{ inputs.targetArch }} - echo "$(pwd)" >> $GITHUB_PATH - export PATH="$PATH:$(pwd)" + label="//cli:cli_${cli_variant}_${GOOS}_${GOARCH}" + bazel build "${label}" + repository_root=$(git rev-parse --show-toplevel) + out_rel=$(bazel cquery --output=files "${label}") + out_loc="$(realpath "${repository_root}/${out_rel}")" + cp "${out_loc}" "${OUTPUT_PATH}" + chmod +w "${OUTPUT_PATH}" + echo "$(dirname "${OUTPUT_PATH}")" >> $GITHUB_PATH + export PATH="$PATH:$(dirname "${OUTPUT_PATH}")" echo "::endgroup::" # TODO: Replace with https://github.com/sigstore/sigstore-installer/tree/initial @@ -80,15 +91,15 @@ runs: COSIGN_PUBLIC_KEY: ${{ inputs.cosignPublicKey }} COSIGN_PRIVATE_KEY: ${{ inputs.cosignPrivateKey }} COSIGN_PASSWORD: ${{ inputs.cosignPassword }} + OUTPUT_PATH: ${{ inputs.outputPath || format('./build/constellation-{0}-{1}', inputs.targetOS, inputs.targetArch) }} run: | - SIGN_TARGET=constellation-${{ inputs.targetOS }}-${{ inputs.targetArch }} echo "$COSIGN_PUBLIC_KEY" > cosign.pub # Enabling experimental mode also publishes signature to Rekor - COSIGN_EXPERIMENTAL=1 cosign sign-blob --key env://COSIGN_PRIVATE_KEY ${SIGN_TARGET} > ${SIGN_TARGET}.sig + COSIGN_EXPERIMENTAL=1 cosign sign-blob --key env://COSIGN_PRIVATE_KEY "${OUTPUT_PATH}" > "${OUTPUT_PATH}.sig" # Verify - As documentation & check # Local Signature (input: artifact, key, signature) - cosign verify-blob --key cosign.pub --signature ${SIGN_TARGET}.sig ${SIGN_TARGET} + cosign verify-blob --key cosign.pub --signature "${OUTPUT_PATH}.sig" "${OUTPUT_PATH}" # Transparency Log Signature (input: artifact, key) - uuid=$(rekor-cli search --artifact ${SIGN_TARGET} | tail -n 1) + uuid=$(rekor-cli search --artifact "${OUTPUT_PATH}" | tail -n 1) sig=$(rekor-cli get --uuid=$uuid --format=json | jq -r .Body.HashedRekordObj.signature.content) - cosign verify-blob --key cosign.pub --signature <(echo $sig) ${SIGN_TARGET} + cosign verify-blob --key cosign.pub --signature <(echo $sig) "${OUTPUT_PATH}" diff --git a/.github/actions/build_debugd/action.yml b/.github/actions/build_debugd/action.yml index 30d203dbc..31492aad8 100644 --- a/.github/actions/build_debugd/action.yml +++ b/.github/actions/build_debugd/action.yml @@ -4,7 +4,7 @@ description: Build the Constellation debugd binary inputs: outputPath: description: "Output path of the binary" - default: "./debugd" + default: "./build/debugd" required: true # Linux runner only (homedir trick does not work on macOS, required for private runner) @@ -13,14 +13,16 @@ runs: steps: - name: Build debugd shell: bash + env: + OUTPUT_PATH: ${{ inputs.outputPath }} run: | echo "::group::Build debugd" - homedir="$(getent passwd $(id -u) | cut -d ":" -f 6)" - mkdir -p build && cd build - cmake -DBAZEL:STRING=bazelisk .. - export GOCACHE=${homedir}/.cache/go-build - export GOPATH=${homedir}/go - export GOMODCACHE=${homedir}/.cache/go-mod - make debugd - mv -n debugd "${{ inputs.outputPath }}" + mkdir -p "$(dirname "${OUTPUT_PATH}")" + label=//debugd/cmd/debugd:debugd_linux_amd64 + bazel build "${label}" + repository_root=$(git rev-parse --show-toplevel) + out_rel=$(bazel cquery --output=files "${label}") + out_loc="$(realpath "${repository_root}/${out_rel}")" + cp "${out_loc}" "${OUTPUT_PATH}" + chmod +w "${OUTPUT_PATH}" echo "::endgroup::" diff --git a/.github/actions/build_disk_mapper/action.yml b/.github/actions/build_disk_mapper/action.yml index fd2208c0c..12c51a38a 100644 --- a/.github/actions/build_disk_mapper/action.yml +++ b/.github/actions/build_disk_mapper/action.yml @@ -4,7 +4,7 @@ description: Build the Constellation disk-mapper binary inputs: outputPath: description: "Output path of the binary" - default: "./disk-mapper" + default: "./build/disk-mapper" required: true # Linux runner only (Docker required) @@ -13,10 +13,16 @@ runs: steps: - name: Build the disk-mapper shell: bash + env: + OUTPUT_PATH: ${{ inputs.outputPath }} run: | echo "::group::Build the disk-mapper" - mkdir -p build && cd build - cmake -DBAZEL:STRING=bazelisk .. - make disk-mapper - mv -n disk-mapper "${{ inputs.outputPath }}" + mkdir -p "$(dirname "${OUTPUT_PATH}")" + label="//disk-mapper/cmd:disk-mapper_linux_amd64" + bazel build "${label}" + repository_root=$(git rev-parse --show-toplevel) + out_rel=$(bazel cquery --output=files "${label}") + out_loc="$(realpath "${repository_root}/${out_rel}")" + cp "${out_loc}" "${OUTPUT_PATH}" + chmod +w "${OUTPUT_PATH}" echo "::endgroup::" diff --git a/.github/actions/build_measurement_reader/action.yml b/.github/actions/build_measurement_reader/action.yml index e586946c8..bec75e762 100644 --- a/.github/actions/build_measurement_reader/action.yml +++ b/.github/actions/build_measurement_reader/action.yml @@ -4,7 +4,7 @@ description: Build the Constellation measurement-reader binary inputs: outputPath: description: "Output path of the binary" - default: "./measurement-reader" + default: "./build/measurement-reader" required: true # Linux runner only (Docker required) @@ -13,10 +13,16 @@ runs: steps: - name: Build the measurement-reader shell: bash + env: + OUTPUT_PATH: ${{ inputs.outputPath }} run: | echo "::group::Build the measurement-reader" - mkdir -p build && cd build - cmake .. - make measurement-reader - mv -n measurement-reader "${{ inputs.outputPath }}" + mkdir -p "$(dirname "${OUTPUT_PATH}")" + label="//measurement-reader/cmd:measurement-reader_linux_amd64" + bazel build "${label}" + repository_root=$(git rev-parse --show-toplevel) + out_rel=$(bazel cquery --output=files "${label}") + out_loc="$(realpath "${repository_root}/${out_rel}")" + cp "${out_loc}" "${OUTPUT_PATH}" + chmod +w "${OUTPUT_PATH}" echo "::endgroup::" diff --git a/.github/actions/build_upgrade_agent/action.yml b/.github/actions/build_upgrade_agent/action.yml index 00f59f6ab..0dd906209 100644 --- a/.github/actions/build_upgrade_agent/action.yml +++ b/.github/actions/build_upgrade_agent/action.yml @@ -4,7 +4,7 @@ description: Build the Constellation upgrade-agent binary inputs: outputPath: description: "Output path of the binary" - default: "./upgrade-agent" + default: "./build/upgrade-agent" required: true # Linux runner only (Docker required) @@ -13,10 +13,16 @@ runs: steps: - name: Build the upgrade-agent shell: bash + env: + OUTPUT_PATH: ${{ inputs.outputPath }} run: | echo "::group::Build the upgrade-agent" - mkdir -p build && cd build - cmake -DBAZEL:STRING=bazelisk .. - make upgrade-agent - mv -n upgrade-agent "${{ inputs.outputPath }}" + mkdir -p "$(dirname "${OUTPUT_PATH}")" + label="//upgrade-agent/cmd:upgrade_agent_linux_amd64" + bazel build "${label}" + repository_root=$(git rev-parse --show-toplevel) + out_rel=$(bazel cquery --output=files "${label}") + out_loc="$(realpath "${repository_root}/${out_rel}")" + cp "${out_loc}" "${OUTPUT_PATH}" + chmod +w "${OUTPUT_PATH}" echo "::endgroup::" diff --git a/.github/actions/e2e_mini/action.yml b/.github/actions/e2e_mini/action.yml index 434072ccd..778ec50e7 100644 --- a/.github/actions/e2e_mini/action.yml +++ b/.github/actions/e2e_mini/action.yml @@ -11,6 +11,9 @@ inputs: azureTenantID: description: "Azure tenant to use for login with OIDC" required: true + buildBuddyApiKey: + description: "BuildBuddy API key for caching Bazel artifacts" + required: true runs: using: "composite" @@ -25,10 +28,17 @@ runs: with: go-version: "1.20.2" + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "true" + buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }} + - name: Build Constellation CLI uses: ./.github/actions/build_cli with: enterpriseCLI: true + outputPath: build/constellation - name: MiniConstellation E2E shell: bash diff --git a/.github/actions/e2e_test/action.yml b/.github/actions/e2e_test/action.yml index b1c9422d9..dabe54f51 100644 --- a/.github/actions/e2e_test/action.yml +++ b/.github/actions/e2e_test/action.yml @@ -71,6 +71,9 @@ inputs: sonobuoyTestSuiteCmd: description: "The sonobuoy test suite to run." required: false + buildBuddyApiKey: + description: "BuildBuddy API key for caching Bazel artifacts" + required: true outputs: kubeconfig: @@ -104,12 +107,19 @@ runs: echo "hostOS=$(go env GOOS)" >> $GITHUB_OUTPUT echo "hostArch=$(go env GOARCH)" >> $GITHUB_OUTPUT + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "true" + buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }} + - name: Build CLI uses: ./.github/actions/build_cli with: targetOS: ${{ steps.determine-build-target.outputs.hostOS }} targetArch: ${{ steps.determine-build-target.outputs.hostArch }} enterpriseCLI: ${{ inputs.keepMeasurements }} + outputPath: "build/constellation" - name: Build the bootstrapper id: build-bootstrapper diff --git a/.github/actions/setup_bazel/action.yml b/.github/actions/setup_bazel/action.yml new file mode 100644 index 000000000..f20db6a86 --- /dev/null +++ b/.github/actions/setup_bazel/action.yml @@ -0,0 +1,55 @@ +name: Setup bazel +description: Setup Bazel for CI builds and tests + +inputs: + useCache: + description: "Cache Bazel artifacts. Use 'true' to enable with rw, 'readonly' to download, and 'false' to disable." + default: "false" + required: true + buildBuddyApiKey: + description: "BuildBuddy API key for caching Bazel artifacts" + required: false + +runs: + using: "composite" + steps: + - name: Check inputs + shell: bash + run: | + echo "::group::Check inputs" + if [[ "${{ inputs.useCache }}" != "true" && "${{ inputs.useCache }}" != "readonly" && "${{ inputs.useCache }}" != "false" ]]; then + echo "Invalid value for 'useCache' input: '${{ inputs.useCache }}'. Must be 'true', 'readonly', or 'false'." + exit 1 + fi + if [[ "${{ inputs.useCache }}" == "true" || "${{ inputs.useCache }}" == "readonly" ]] && [[ -z "${{ inputs.buildBuddyApiKey }}" ]]; then + echo "BuildBuddy API key is required when cache is enabled." + exit 1 + fi + echo "::endgroup::" + - name: Configure Bazel + shell: bash + if: inputs.useCache == 'true' || inputs.useCache == 'readonly' + env: + BUILDBUDDY_ORG_API_KEY: ${{ inputs.buildBuddyApiKey }} + run: | + echo "::group::Configure Bazel" + cat <> ~/.bazelrc + build --bes_results_url=https://app.buildbuddy.io/invocation/ + build --bes_backend=grpcs://remote.buildbuddy.io + build --remote_cache=grpcs://remote.buildbuddy.io + build --remote_timeout=3600 + build --experimental_remote_build_event_upload=minimal + build --experimental_remote_cache_compression + build --remote_header=x-buildbuddy-api-key=${BUILDBUDDY_ORG_API_KEY} + EOF + echo "::endgroup::" + - name: Configure Bazel (readonly) + shell: bash + if: inputs.useCache == 'readonly' + run: | + echo "::group::Configure Bazel (readonly)" + echo "build --remote_upload_local_results=false" >> ~/.bazelrc + echo "::endgroup::" + - name: Check bazel version + shell: bash + run: bazel version diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index a7b01ec86..057091f72 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -26,6 +26,12 @@ jobs: with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "true" + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + - name: Build the bootstrapper uses: ./.github/actions/build_bootstrapper @@ -37,6 +43,12 @@ jobs: with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "true" + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + - name: Build debugd uses: ./.github/actions/build_debugd @@ -48,17 +60,25 @@ jobs: with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "true" + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + - name: Build cdbg (Linux, amd64) uses: ./.github/actions/build_cdbg with: targetOS: "linux" targetArch: "amd64" + outputPath: ./build/cdbg_linux_amd64 - name: Build cdbg (Linux, arm64) uses: ./.github/actions/build_cdbg with: targetOS: "linux" targetArch: "arm64" + outputPath: ./build/cdbg_linux_arm64 build-cdbg-macos: runs-on: ubuntu-22.04 @@ -68,17 +88,25 @@ jobs: with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "true" + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + - name: Build cdbg (macOS, amd64) uses: ./.github/actions/build_cdbg with: targetOS: "darwin" targetArch: "amd64" + outputPath: ./build/cdbg_darwin_amd64 - name: Build cdbg (macOS, arm64) uses: ./.github/actions/build_cdbg with: targetOS: "darwin" targetArch: "arm64" + outputPath: ./build/cdbg_darwin_arm64 build-disk-mapper: runs-on: ubuntu-22.04 @@ -88,6 +116,12 @@ jobs: with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "true" + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + - name: Build disk-mapper uses: ./.github/actions/build_disk_mapper @@ -99,6 +133,12 @@ jobs: with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "true" + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + - name: Build measurement-reader uses: ./.github/actions/build_measurement_reader @@ -110,6 +150,12 @@ jobs: with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "true" + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + - name: Build CLI (Linux, amd64) uses: ./.github/actions/build_cli with: @@ -130,6 +176,12 @@ jobs: with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "true" + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + - name: Build CLI (macOS, amd64) uses: ./.github/actions/build_cli with: diff --git a/.github/workflows/build-os-image.yml b/.github/workflows/build-os-image.yml index 2fcb53250..f1fe238ff 100644 --- a/.github/workflows/build-os-image.yml +++ b/.github/workflows/build-os-image.yml @@ -68,6 +68,11 @@ jobs: go-version: "1.20.2" cache: true + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "false" + - name: Build bootstrapper if: inputs.stream != 'debug' uses: ./.github/actions/build_bootstrapper diff --git a/.github/workflows/e2e-mini.yml b/.github/workflows/e2e-mini.yml index 4ef736184..536f1a462 100644 --- a/.github/workflows/e2e-mini.yml +++ b/.github/workflows/e2e-mini.yml @@ -41,3 +41,4 @@ jobs: azureClientID: ${{ secrets.AZURE_E2E_MINI_CLIENT_ID }} azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} azureTenantID: ${{ secrets.AZURE_TENANT_ID }} + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} diff --git a/.github/workflows/e2e-test-daily.yml b/.github/workflows/e2e-test-daily.yml index b02781e79..7ebf81578 100644 --- a/.github/workflows/e2e-test-daily.yml +++ b/.github/workflows/e2e-test-daily.yml @@ -96,6 +96,7 @@ jobs: gcp_service_account: "constellation-e2e@constellation-331613.iam.gserviceaccount.com" gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }} test: "sonobuoy full" + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} - name: Always terminate cluster if: always() diff --git a/.github/workflows/e2e-test-manual.yml b/.github/workflows/e2e-test-manual.yml index b04c120fd..90360d3ee 100644 --- a/.github/workflows/e2e-test-manual.yml +++ b/.github/workflows/e2e-test-manual.yml @@ -258,6 +258,7 @@ jobs: azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }} osImage: ${{ needs.find-latest-image.outputs.image }} isDebugImage: ${{ needs.find-latest-image.outputs.isDebugImage }} + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} - name: Always terminate cluster if: always() diff --git a/.github/workflows/e2e-test-weekly.yml b/.github/workflows/e2e-test-weekly.yml index c0435b628..128a818eb 100644 --- a/.github/workflows/e2e-test-weekly.yml +++ b/.github/workflows/e2e-test-weekly.yml @@ -148,6 +148,7 @@ jobs: gcp_service_account: "constellation-e2e@constellation-331613.iam.gserviceaccount.com" gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }} test: ${{ matrix.test }} + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} - name: Always terminate cluster if: always() diff --git a/.github/workflows/release-cli.yml b/.github/workflows/release-cli.yml index 1372d41ec..afaadc5cd 100644 --- a/.github/workflows/release-cli.yml +++ b/.github/workflows/release-cli.yml @@ -29,6 +29,11 @@ jobs: with: ref: ${{ inputs.ref || github.head_ref }} + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "false" + - name: Build CLI uses: ./.github/actions/build_cli with: @@ -107,6 +112,11 @@ jobs: - name: Download Syft & Grype uses: ./.github/actions/install_syft_grype + - name: Setup bazel + uses: ./.github/actions/setup_bazel + with: + useCache: "false" + # Build one CLI since Syft's go-module catalog will default to binary parsing. # Binary parsing has the advantage that it will not include other dependencies from our repo not included in the CLI. # This seems to work fine for one OS & one arch as long as we don't have OS specific imports. diff --git a/dev-docs/workflows/build-test-run.md b/dev-docs/workflows/build-test-run.md index d1fffdd5c..e30d9e285 100644 --- a/dev-docs/workflows/build-test-run.md +++ b/dev-docs/workflows/build-test-run.md @@ -50,6 +50,17 @@ bazel build //cli:cli_oss_darwin_amd64 # cross compile CLI for mac amd64 bazel build //cli:cli_oss_darwin_arm64 # cross compile CLI for mac arm64 ``` +## Remote caching and execution + +We use BuildBuddy for remote caching (and maybe remote execution in the future). To use it, you need to join the BuildBuddy organization and get an API key. Then, you can write it to `~/.bazelrc`: + +``` +build --remote_header=x-buildbuddy-api-key= +``` + +To use the remote cache, build the project with `bazel build --config remote_cache //path/to:target`. +You can also copy the `remote_cache` config from `.bazelrc` to your `~/.bazelrc` and remove the `remote_cache` prefix to make it the default. + # Test You can run all integration and unitttests like this: diff --git a/tools/workspace_status.sh b/tools/workspace_status.sh index 1f8f82a17..670733ae7 100755 --- a/tools/workspace_status.sh +++ b/tools/workspace_status.sh @@ -22,6 +22,7 @@ timestamp() { "${REPOSITORY_ROOT}/tools/pseudo-version" -print-timestamp -timestamp-format '2006-01-02T15:04:05Z07:00' } +echo "REPO_URL https://github.com/edgelesssys/constellation.git" echo "STABLE_STAMP_COMMIT $(git rev-parse HEAD)" echo "STABLE_STAMP_STATE $(git diff-index --quiet HEAD -- && echo "clean" || echo "dirty")" echo "STABLE_STAMP_VERSION $(pseudo_version)"