e2e: measurements reproducibility test for images (#3654)

.github/actions/check_measurements_reproducibility/action.yml

@@ -0,0 +1,58 @@
+name: Check measurements reproducibility
+description: Check if the measurements of a given release are reproducible.
+
+inputs:
+  version:
+    type: string
+    description: The version of the measurements that are downloaded from the CDN.
+    required: true
+  ref:
+    type: string
+    description: The git ref to check out. You probably want this to be the tag of the release you are testing.
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        ref: ${{ inputs.ref }}
+        path: ./release
+
+    - name: Set up bazel
+      uses: ./.github/actions/setup_bazel_nix
+      with:
+        useCache: "false"
+        nixTools: |
+          systemdUkify
+          jq
+          jd-diff-patch
+          moreutils
+
+    - name: Build images
+      id: build-images
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        # Build required binaries
+        pushd release
+        bazel build //image/system:stable
+        echo "buildPath=$PWD/bazel-bin/image" | tee -a "$GITHUB_OUTPUT"
+        popd
+
+    - name: Download measurements
+      shell: bash
+      run: |
+        curl -fsLO https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/${{ inputs.version }}/image/measurements.json
+    
+    - name: Cleanup release measurements and generate our own
+      shell: bash
+      run: |
+        ${{ github.action_path }}/create_measurements.sh "${{ steps.build-images.outputs.buildPath }}"
+        
+    - name: Compare measurements
+      shell: bash
+      run: |
+        ${{ github.action_path }}/compare_measurements.sh "${{ steps.build-images.outputs.buildPath }}"

.github/actions/check_measurements_reproducibility/compare_measurements.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# no -e since we need to collect errors later
+# no -u since it interferes with checking associative arrays
+set -o pipefail
+shopt -s extglob
+
+declare -A errors
+
+for directory in "$1"/system/!(mkosi_wrapper.sh); do
+  dirname="$(basename "$directory")"
+  attestationVariant="$(echo "$dirname" | cut -d_ -f2)"
+
+  echo "Their measurements for $attestationVariant:"
+  ts "  " < "$attestationVariant"_their-measurements.json
+  echo "Own measurements for $attestationVariant:"
+  ts "  " < "$attestationVariant"_own-measurements.json
+
+  diff="$(jd ./"$attestationVariant"_their-measurements.json ./"$attestationVariant"_own-measurements.json)"
+  if [[ -n $diff ]]; then
+    errors["$attestationVariant"]="$diff"
+  fi
+done
+
+for attestationVariant in "${!errors[@]}"; do
+  echo "Failed to reproduce measurements for $attestationVariant:"
+  echo "${errors["$attestationVariant"]}" | ts "  "
+done
+
+if [[ ${#errors[@]} -ne 0 ]]; then
+  exit 1
+fi

.github/actions/check_measurements_reproducibility/create_measurements.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+shopt -s extglob
+
+for directory in "$1"/system/!(mkosi_wrapper.sh); do
+  dirname="$(basename "$directory")"
+  csp="$(echo "$dirname" | cut -d_ -f1)"
+  attestationVariant="$(echo "$dirname" | cut -d_ -f2)"
+
+  # This jq filter selects the measurements for the correct CSP and attestation variant
+  # and then removes all `warnOnly: true` measurements.
+  jq --arg attestation_variant "$attestationVariant" --arg csp "$csp" \
+    '
+      .list.[]
+      | select(
+        .attestationVariant == $attestation_variant
+        and (.csp | ascii_downcase) == $csp
+      )
+      | .measurements
+      | to_entries
+      | map(select(.value.warnOnly | not))
+      | from_entries
+      | del(.[] .warnOnly)
+  ' \
+    measurements.json > "$attestationVariant"_their-measurements.json
+
+  bazel run --run_under "sudo --preserve-env" //image/measured-boot/cmd -- "$directory/constellation" /dev/stdout | jq '.measurements' > ./"$attestationVariant"_own-measurements.json
+done

.github/workflows/check-measurements-reproducibility.yml

@@ -0,0 +1,25 @@
+name: Check measurements reproducibility
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        type: string
+        description: The version of the measurements that are downloaded from the CDN.
+        required: true
+      ref:
+        type: string
+        description: The git ref to check out. You probably want this to be the tag of the release you are testing.
+        required: true
+
+jobs:
+  check-reproducibility:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        
+      - name: Check reproducibility
+        uses: ./.github/actions/check_measurements_reproducibility
+        with:
+          version: ${{ github.event.inputs.version }}
+          ref: ${{ github.event.inputs.ref }}

.github/workflows/release.yml

@@ -239,6 +239,17 @@ jobs:
       stream: "stable"
       ref: ${{ needs.verify-inputs.outputs.WORKING_BRANCH }}
 
+  check-measurements-reproducibility:
+    name: Check measurements reproducibility
+    needs: [verify-inputs, os-image]
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check reproducibility
+        uses: ./.github/actions/check_measurements_reproducibility
+        with:
+          version: ${{ inputs.version }}
+          ref: ${{ needs.verify-inputs.outputs.WORKING_BRANCH }}
+
   update-hardcoded-measurements:
     name: Update hardcoded measurements (in the CLI)
     needs: [verify-inputs, os-image]