ci: use rbe for unit tests

.bazelrc

@@ -47,7 +47,6 @@ common:remote_cache --bes_backend=grpcs://remote.buildbuddy.io
 common:remote_cache --remote_cache=grpcs://remote.buildbuddy.io
 common:remote_cache --remote_timeout=3600
 common:remote_cache --experimental_remote_build_event_upload=minimal
-common:remote_cache --experimental_remote_cache_compression
 common:remote_cache --nolegacy_important_outputs
 common:remote_cache_readonly --noremote_upload_local_results # Uploads logs & artifacts without writing to cache
 

.github/actionlint.yaml

@@ -1,3 +1,10 @@
 self-hosted-runner:
   # Labels of self-hosted runner in array of string
-  labels: [azure-cvm, bazel-cached, bazel-nocache, ubuntu-latest-8-cores]
+  labels:
+    [
+      azure-cvm,
+      arc-runner-set,
+      bazel-cached,
+      bazel-nocache,
+      ubuntu-latest-8-cores,
+    ]

.github/actions/setup_bazel_nix/action.yml

@@ -3,12 +3,15 @@ description: Setup Bazel and Nix for CI builds and tests
 
 inputs:
   useCache:
-    description: "Cache Bazel artifacts. Use 'true' to enable with rw, 'readonly' to download, 'log' to disable cache but upload logs, and 'false' to disable."
+    description: "Cache Bazel artifacts. Use 'true' to enable with rw, 'readonly' to download, 'rbe' to enable with remote execution, 'log' to disable cache but upload logs, and 'false' to disable."
     default: "false"
     required: true
   buildBuddyApiKey:
     description: "BuildBuddy API key for caching Bazel artifacts"
     required: false
+  rbePlatform:
+    description: "RBE platform to use. If empty, RBE will not be used."
+    required: false
 
 runs:
   using: "composite"
@@ -18,7 +21,7 @@ runs:
       shell: bash
       run: |
         echo "::group::Check inputs"
-        if [[ "${{ inputs.useCache }}" != "true" && "${{ inputs.useCache }}" != "readonly" && "${{ inputs.useCache }}" != "logs" && "${{ inputs.useCache }}" != "false" ]]; then
+        if [[ "${{ inputs.useCache }}" != "true" && "${{ inputs.useCache }}" != "readonly" && "${{ inputs.useCache }}" != "rbe" && "${{ inputs.useCache }}" != "logs" && "${{ inputs.useCache }}" != "false" ]]; then
           echo "Invalid value for 'useCache' input: '${{ inputs.useCache }}'. Must be 'true', 'readonly', or 'false'."
           exit 1
         fi
@@ -26,6 +29,22 @@ runs:
           echo "BuildBuddy API key is required when cache is enabled."
           exit 1
         fi
+        if [[ "${{ inputs.useCache }}" == "rbe" && -z "${{ inputs.rbePlatform }}" ]]; then
+          echo "RBE platform is required when cache is enabled."
+          exit 1
+        fi
+        if [[ -n "${{inputs.rbePlatform}}" ]]; then
+        case "${{ inputs.rbePlatform }}" in
+          ubuntu-22.04)
+            echo "rbeConfig=build_barn_rbe_ubuntu_22_04" | tee -a "$GITHUB_OUTPUT"
+            ;;
+
+          *)
+            echo "Invalid value for 'rbePlatform' input: '${{ inputs.rbePlatform }}'. Must be 'ubuntu-22.04'."
+            exit 1
+            ;;
+        esac
+        fi
         if command -v nix; then
           echo "nixPreinstalled=true" | tee -a "$GITHUB_OUTPUT"
         else
@@ -61,6 +80,35 @@ runs:
         fi
         echo "::endgroup::"
 
+    - name: Prepare to install tools
+      shell: bash
+      run: |
+        echo "::group::Prepare to install nix and bazel"
+        requiredTools=( "curl" "xz" "unzip" "git" )
+        declare -A packageNamesUbuntu=( ["curl"]="curl" ["xz"]="xz-utils" ["unzip"]="unzip" ["git"]="git" )
+        missingTools=()
+        for tool in "${requiredTools[@]}"; do
+          if ! command -v "$tool"; then
+            echo "$tool not found, installing..."
+            missingTools+=("$tool")
+          else
+            echo "$tool found $(command -v "$tool")"
+          fi
+        done
+        missingPackagesUbuntu=()
+        for tool in "${missingTools[@]}"; do
+          echo "Ubuntu name for $tool is ${packageNamesUbuntu[$tool]}"
+          missingPackagesUbuntu+=("${packageNamesUbuntu[$tool]}")
+        done
+        if [[ "${#missingTools[@]}" -gt 0 ]]; then
+          echo "Installing missing tools ${missingTools[*]}..."
+          if [[ "$RUNNER_OS" == "Linux" ]]; then
+            sudo apt-get update || true
+            sudo apt-get install -y ${missingPackagesUbuntu[*]} || true
+          fi
+        fi
+        echo "::endgroup::"
+
     - name: Install nix
       if: steps.check_inputs.outputs.nixPreinstalled == 'false'
       uses: cachix/install-nix-action@6a9a9e84a173d90b3ffb42c5ddaf9ea033fad011 # v23
@@ -74,12 +122,6 @@ runs:
         fi
         echo "::endgroup::"
 
-
-    - uses: cachix/cachix-action@6a9a34cdd93d0ae4b4b59fd678660efb08109f2f # v12
-      with:
-        name: katexochen
-        extraPullNames: nix-community
-
     - name: Install Bazelisk
       if: steps.check_inputs.outputs.bazelPreinstalled == 'false' && steps.check_inputs.outputs.nixOS == 'false'
       shell: bash
@@ -88,6 +130,8 @@ runs:
         ARCH: ${{ steps.check_inputs.outputs.arch }}
       run: |
         echo "::group::Install Bazelisk"
+        sudo mkdir -p /usr/local/bin
+        sudo chown -R "$USER" /usr/local/bin
         curl -fsSLo /usr/local/bin/bazel "https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-${OS}-${ARCH}"
         chmod +x /usr/local/bin/bazel
         echo "::endgroup::"
@@ -180,6 +224,39 @@ runs:
         EOF
         echo "::endgroup::"
 
+    - name: Configure Bazel (rbe)
+      if: inputs.useCache == 'rbe'
+      shell: bash
+      env:
+        RBE_CONFIG: ${{ steps.check_inputs.outputs.rbeConfig }}
+        WORKSPACE: ${{ github.workspace }}
+      run: |
+        echo "::group::Configure Bazel"
+        cat <<EOF >> "${WORKSPACE}/.bazeloverwriterc"
+        common --config=${RBE_CONFIG}
+        common --repository_cache=/repository_cache
+        common --repo_env=GOPROXY=http://goproxy:3000
+        EOF
+        echo "::endgroup::"
+    - name: Configure Bazel (rbe logs)
+      if: inputs.useCache == 'rbe' && inputs.buildBuddyApiKey != ''
+      shell: bash
+      env:
+        BUILDBUDDY_ORG_API_KEY: ${{ inputs.buildBuddyApiKey }}
+        WORKSPACE: ${{ github.workspace }}
+      run: |
+        echo "::group::Configure Bazel"
+        cat <<EOF >> "${WORKSPACE}/.bazeloverwriterc"
+        common --bes_results_url=https://app.buildbuddy.io/invocation/
+        common --bes_backend=grpcs://remote.buildbuddy.io
+        common --remote_header=x-buildbuddy-api-key=${BUILDBUDDY_ORG_API_KEY}
+        cquery --bes_results_url=
+        cquery --bes_backend=
+        query --bes_results_url=
+        query --bes_backend=
+        EOF
+        echo "::endgroup::"
+
     - name: Disable disk cache on GitHub Actions runners
       if: startsWith(runner.name , 'GitHub Actions')
       shell: bash

.github/workflows/build-binaries.yml

@@ -19,7 +19,7 @@ on:
 
 jobs:
   build-binaries:
-    runs-on: [self-hosted, bazel-cached]
+    runs-on: [arc-runner-set]
     steps:
       - name: Checkout
         uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
@@ -29,7 +29,8 @@ jobs:
       - name: Setup bazel
         uses: ./.github/actions/setup_bazel_nix
         with:
-          useCache: "true"
+          useCache: "rbe"
+          rbePlatform: "ubuntu-22.04"
           buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
 
       - name: Build all
@@ -44,6 +45,7 @@ jobs:
 
         run: |
           bazel build \
+            --remote_download_minimal \
             "${bootstrapper}" \
             "${debugd}" \
             "${cdbg}" \

.github/workflows/test-tidy.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   tidycheck:
     name: tidy, check and generate
-    runs-on: [self-hosted, bazel-cached]
+    runs-on: [arc-runner-set]
     permissions:
       id-token: write
       contents: read
@@ -32,7 +32,8 @@ jobs:
       - name: Setup Bazel
         uses: ./.github/actions/setup_bazel_nix
         with:
-          useCache: "true"
+          useCache: "rbe"
+          rbePlatform: "ubuntu-22.04"
           buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
 
       - name: Assume AWS role to upload Bazel dependencies to S3

.github/workflows/test-unittest.yml

@@ -23,7 +23,7 @@ on:
 
 jobs:
   test-unittest:
-    runs-on: [self-hosted, bazel-cached]
+    runs-on: [arc-runner-set]
     permissions:
       id-token: write
       contents: read
@@ -35,6 +35,13 @@ jobs:
           ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
           fetch-depth: 0
 
+      - name: Setup bazel
+        uses: ./.github/actions/setup_bazel_nix
+        with:
+          useCache: "rbe"
+          rbePlatform: "ubuntu-22.04"
+          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+
       - name: Install AWS cli
         run: |
           curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
@@ -48,21 +55,15 @@ jobs:
           role-to-assume: arn:aws:iam::795746500882:role/GithubActionGocoverage
           aws-region: eu-central-1
 
-      - name: Setup bazel
-        uses: ./.github/actions/setup_bazel_nix
-        with:
-          useCache: "true"
-          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
-
       - name: Unit Tests
         env:
           TMPDIR: ${{ runner.temp }}
-        run: bazel test //... --test_output=errors --config=nostamp
+        run: bazel test //... --test_output=errors --config=nostamp --remote_download_minimal
 
       - name: Coverage
         id: coverage
         run: |
-          bazel run //bazel/ci:gocoverage_diff
+          bazel run //bazel/ci:gocoverage_diff --config=nostamp
           lines=$(wc -l < coverage_diff.md)
           uploadable=$([[ ${lines} -gt 3 ]] && echo "true" || echo "false")
           echo "uploadable=$uploadable" | tee -a "$GITHUB_OUTPUT"

bazel/bazelrc/ci.bazelrc

@@ -77,6 +77,4 @@ common --bes_timeout=600s
 
 # options to limit the amount of data uploaded/downloaded to/from the remote cache
 common --experimental_remote_build_event_upload=minimal
-common --experimental_remote_cache_compression
 common --nolegacy_important_outputs
-common --noremote_upload_local_results