Add new generate measurements matrix CI/CD action (now with AWS support) (#641)

.github/actions/constellation_measure/action.yml

@@ -1,133 +0,0 @@
-name: Constellation measure
-description: |
-  Create measurements of a Constellation cluster and print to stdout.
-  Optionally sign and/or upload to S3, if corresponding inputs are provided.
-inputs:
-  cloudProvider:
-    description: "Either 'gcp' or 'azure'."
-    required: true
-  cosignPublicKey:
-    description: "Cosign public key"
-    required: false
-    default: ""
-  cosignPrivateKey:
-    description: "Cosign private key"
-    required: false
-    default: ""
-  cosignPassword:
-    description: "Password for Cosign private key"
-    required: false
-    default: ""
-  awsAccessKeyID:
-    description: "AWS access key ID to upload measurements"
-    required: false
-    default: ""
-  awsSecretAccessKey:
-    description: "AWS secret access key to upload measurements"
-    required: false
-    default: ""
-  awsDefaultRegion:
-    description: "AWS region of S3 bucket to upload measurements"
-    required: false
-    default: ""
-  awsBucketName:
-    description: "S3 bucket name to upload measurements to"
-    required: false
-    default: ""
-runs:
-  using: "composite"
-  steps:
-    - name: Build hack/pcr-reader
-      run: |
-        echo "::group::Build pcr-reader"
-        go build .
-        echo "$(pwd)" >> $GITHUB_PATH
-      working-directory: hack/pcr-reader
-      shell: bash
-
-    # Check /docs/secure_software_distribution.md#sign-measurements
-    # for why we ignore certain measurement values.
-    - name: Fetch PCRs
-      run: |
-        KUBECONFIG="$PWD/constellation-admin.conf" kubectl rollout status ds/verification-service -n kube-system --timeout=3m
-        CONSTELL_IP=$(jq -r ".ip" constellation-id.json)
-        pcr-reader --constell-ip ${CONSTELL_IP} -format json > measurements.json
-        case $CSP in
-          azure)
-            yq e 'del(.[0,6,10,16,17,18,19,20,21,22,23])' -I 0 -o json -i measurements.json
-            yq '.4.warnOnly = false |
-              .8.warnOnly = false |
-              .9.warnOnly = false |
-              .11.warnOnly = false |
-              .12.warnOnly = false |
-              .13.warnOnly = false |
-              .15.warnOnly = false |
-              .15.expected = "0000000000000000000000000000000000000000000000000000000000000000"' \
-              -I 0 -o json -i measurements.json
-            ;;
-          gcp)
-            yq e 'del(.[16,17,18,19,20,21,22,23])' -I 0 -o json -i measurements.json
-            yq '.0.warnOnly = false |
-              .4.warnOnly = false |
-              .8.warnOnly = false |
-              .9.warnOnly = false |
-              .11.warnOnly = false |
-              .12.warnOnly = false |
-              .13.warnOnly = false |
-              .15.warnOnly = false |
-              .15.expected = "0000000000000000000000000000000000000000000000000000000000000000"' \
-              -I 0 -o json -i measurements.json
-            ;;
-        esac
-        cat measurements.json
-      shell: bash
-      env:
-        CSP: ${{ inputs.cloudProvider }}
-
-    # TODO: Replace with https://github.com/sigstore/sigstore-installer/tree/initial
-    # once it has the functionality
-    - name: Install Cosign
-      uses: sigstore/cosign-installer@9becc617647dfa20ae7b1151972e9b3a2c338a2b # tag=v2.8.1
-      if: ${{ inputs.cosignPublicKey != '' && inputs.cosignPrivateKey != '' && inputs.cosignPassword != '' }}
-    - name: Install Rekor
-      run: |
-        curl -sLO https://github.com/sigstore/rekor/releases/download/v0.12.0/rekor-cli-linux-amd64
-        sudo install rekor-cli-linux-amd64 /usr/local/bin/rekor-cli
-        rm rekor-cli-linux-amd64
-      shell: bash
-      if: ${{ inputs.cosignPublicKey != '' && inputs.cosignPrivateKey != '' && inputs.cosignPassword != '' }}
-    - name: Sign measurements
-      run: |
-        echo "$COSIGN_PUBLIC_KEY" > cosign.pub
-        # Enabling experimental mode also publishes signature to Rekor
-        COSIGN_EXPERIMENTAL=1 cosign sign-blob --key env://COSIGN_PRIVATE_KEY measurements.json > measurements.json.sig
-        # Verify - As documentation & check
-        # Local Signature (input: artifact, key, signature)
-        cosign verify-blob --key cosign.pub --signature measurements.json.sig measurements.json
-        # Transparency Log Signature (input: artifact, key)
-        uuid=$(rekor-cli search --artifact measurements.json | tail -n 1)
-        sig=$(rekor-cli get --uuid=$uuid --format=json | jq -r .Body.HashedRekordObj.signature.content)
-        cosign verify-blob --key cosign.pub --signature <(echo $sig) measurements.json
-      shell: bash
-      env:
-        COSIGN_PUBLIC_KEY: ${{ inputs.cosignPublicKey }}
-        COSIGN_PRIVATE_KEY: ${{ inputs.cosignPrivateKey }}
-        COSIGN_PASSWORD: ${{ inputs.cosignPassword }}
-      if: ${{ inputs.cosignPublicKey != '' && inputs.cosignPrivateKey != '' && inputs.cosignPassword != '' }}
-
-    - name: Upload to S3
-      run: |
-        IMAGE=$(yq e ".provider.${CSP}.image" constellation-conf.yaml)
-        S3_PATH=s3://${PUBLIC_BUCKET_NAME}/${IMAGE,,}
-        aws s3 cp measurements.json ${S3_PATH}/measurements.json
-        if test -f measurements.json.sig; then
-          aws s3 cp measurements.json.sig ${S3_PATH}/measurements.json.sig
-        fi
-      shell: bash
-      env:
-        AWS_ACCESS_KEY_ID: ${{ inputs.awsAccessKeyID }}
-        AWS_SECRET_ACCESS_KEY: ${{ inputs.awsSecretAccessKey }}
-        AWS_DEFAULT_REGION: ${{ inputs.awsDefaultRegion }}
-        PUBLIC_BUCKET_NAME: ${{ inputs.awsBucketName }}
-        CSP: ${{ inputs.cloudProvider }}
-      if: ${{ inputs.awsAccessKeyID != '' && inputs.awsSecretAccessKey != '' && inputs.awsDefaultRegion != '' && inputs.awsBucketName != '' }}

.github/actions/generate_measurements/action.yml

@@ -1,132 +0,0 @@
-name: Generate measurements
-description: "Generates measurements for a specific image"
-inputs:
-  cloudProvider:
-    description: "Which cloud provider to use."
-    required: true
-  osImage:
-    description: "OS image to run."
-    required: true
-  isDebugImage:
-    description: "Is OS img a debug img?"
-    required: true
-  workerNodesCount:
-    description: "Number of worker nodes to spawn."
-    required: false
-    default: "1"
-  controlNodesCount:
-    description: "Number of control-plane nodes to spawn."
-    required: false
-    default: "1"
-  machineType:
-    description: "VM machine type. Make sure it matches selected cloud provider!"
-    required: false
-  kubernetesVersion:
-    description: "Kubernetes version to create the cluster from."
-    required: false
-    default: "1.23"
-  gcpProject:
-    description: "The GCP project to deploy Constellation in."
-    required: false
-  gcp_service_account_json:
-    description: "Service account with permissions to create Constellation on GCP."
-    required: false
-  gcpClusterServiceAccountKey:
-    description: "Service account to use inside the created Constellation cluster on GCP."
-    required: false
-  azureSubscription:
-    description: "The Azure subscription ID to deploy Constellation in."
-    required: false
-  azureTenant:
-    description: "The Azure tenant ID to deploy Constellation in."
-    required: false
-  azureClientID:
-    description: "The client ID of the application registration created for Constellation in Azure."
-    required: false
-  azureClientSecret:
-    description: "The client secret value of the used secret"
-    required: false
-  azureResourceGroup:
-    description: "The resource group to use"
-    required: false
-  azureUserAssignedIdentity:
-    description: "The Azure user assigned identity to use for Constellation."
-    required: false
-  cosignPublicKey:
-    description: "Cosign public key to sign measurements."
-    required: true
-  cosignPrivateKey:
-    description: "Cosign private key to sign measurements."
-    required: true
-  cosignPassword:
-    description: "Cosign password for private key."
-    required: true
-  awsAccessKeyID:
-    description: "AWS access key ID to upload measurements."
-    required: true
-  awsSecretAccessKey:
-    description: "AWS secrets access key to upload measurements."
-    required: true
-  awsDefaultRegion:
-    description: "AWS region of S3 bucket. to upload measurements."
-    required: true
-  awsBucketName:
-    description: "AWS S3 bucket name to upload measurements."
-    required: true
-
-outputs:
-  kubeconfig:
-    description: "Kubeconfig file of the created cluster."
-    value: ${{ steps.create-cluster.outputs.kubeconfig }}
-
-runs:
-  using: "composite"
-  steps:
-    - name: Build CLI
-      uses: ./.github/actions/build_cli
-    - name: Build the bootstrapper
-      id: build-bootstrapper
-      uses: ./.github/actions/build_bootstrapper
-      if: ${{ inputs.isDebugImage == 'true' }}
-    - name: Build cdbg
-      id: build-cdbg
-      uses: ./.github/actions/build_cdbg
-      if: ${{ inputs.isDebugImage == 'true' }}
-
-    - name: Login to GCP
-      uses: ./.github/actions/login_gcp
-      with:
-        gcp_service_account_json: ${{ inputs.gcp_service_account_json }}
-      if: ${{ inputs.cloudProvider == 'gcp' }}
-
-    - name: Create cluster
-      id: create-cluster
-      uses: ./.github/actions/constellation_create
-      with:
-        cloudProvider: ${{ inputs.cloudProvider }}
-        gcpProject: ${{ inputs.gcpProject }}
-        gcpClusterServiceAccountKey: ${{ inputs.gcpClusterServiceAccountKey }}
-        workerNodesCount: ${{ inputs.workerNodesCount }}
-        controlNodesCount: ${{ inputs.controlNodesCount }}
-        machineType: ${{ inputs.machineType }}
-        osImage: ${{ inputs.osImage }}
-        isDebugImage: ${{ inputs.isDebugImage }}
-        kubernetesVersion: ${{ inputs.kubernetesVersion }}
-        azureSubscription: ${{ inputs.azureSubscription }}
-        azureTenant: ${{ inputs.azureTenant }}
-        azureClientID: ${{ inputs.azureClientID }}
-        azureClientSecret: ${{ inputs.azureClientSecret }}
-        azureUserAssignedIdentity: ${{ inputs.azureUserAssignedIdentity }}
-        azureResourceGroup: ${{ inputs.azureResourceGroup }}
-
-    - name: Measure cluster
-      uses: ./.github/actions/constellation_measure
-      with:
-        cloudProvider: ${{ inputs.cloudProvider }}
-        cosignPublicKey: ${{ inputs.cosignPublicKey }}
-        cosignPrivateKey: ${{ inputs.cosignPrivateKey }}
-        cosignPassword: ${{ inputs.cosignPassword }}
-        awsAccessKeyID: ${{ inputs.awsAccessKeyID }}
-        awsSecretAccessKey: ${{ inputs.awsSecretAccessKey }}
-        awsDefaultRegion: ${{ inputs.awsDefaultRegion }}
-        awsBucketName: ${{ inputs.awsBucketName }}

.github/docs/release.md

@@ -87,11 +87,10 @@ This checklist will prepare `v1.3.0` from `v1.2.0`. Adjust your version numbers
         gh workflow run e2e-test-manual-macos.yml --ref release/v$minor -F cloudProvider=gcp -F machineType=n2d-standard-4 -F test="sonobuoy full" -F osImage=v$ver -F isDebugImage=false
         ```
 
-    12. [Generate measurements](/.github/workflows/generate-measurements.yml) for the images on each CSP.
+    12. [Generate measurements](/.github/workflows/generate-measurements.yml) for the images.
 
         ```sh
-           gh workflow run generate-measurements.yml --ref release/v$minor -F cloudProvider=azure -F osImage=v$ver -F isDebugImage=false
+           gh workflow run generate-measurements.yml --ref release/v$minor -F osImage=v$ver -F isDebugImage=false -F signMeasurements=true
-           gh workflow run generate-measurements.yml --ref release/v$minor -F cloudProvider=gcp -F osImage=v$ver -F isDebugImage=false
         ```
 
     13. Create a new tag on this release branch
@@ -142,11 +141,3 @@ This checklist will prepare `v1.3.0` from `v1.2.0`. Adjust your version numbers
     ```
 
 10. Test Constellation mini up
-
-11. Upload AWS measurements to S3 bucket:
-    * Create an AWS cluster using the released version.
-    * Use `hack/pcr-reader` to download measurements.
-    * Create a new folder named after each AWS AMI in [S3 public bucket](https://s3.console.aws.amazon.com/s3/buckets/public-edgeless-constellation?region=us-east-2&tab=objects).
-    * Keep measurements: 4, 8, 9, 11, 12, 13.
-    * Sign the measurements using `cosign sign-blob`.
-    * Upload both `measurements.yaml` & `measurements.yaml.sig` to each created folder in S3.

.github/workflows/generate-measurements.yml

@@ -1,16 +1,8 @@
-name: Generate measurements manually
+name: Generate and Upload Measurements
 
 on:
   workflow_dispatch:
     inputs:
-      cloudProvider:
-        description: "Which cloud provider to use."
-        type: choice
-        options:
-          - "azure"
-          - "gcp"
-        default: "gcp"
-        required: true
       osImage:
         description: "Full name of OS image (CSP independent image version UID)."
         type: string
@@ -19,48 +11,73 @@ on:
         description: "Is OS image a debug image?"
         type: boolean
         required: true
-
+      signMeasurements:
-env:
+        description: "Sign and upload the measurements?"
-  ARM_CLIENT_ID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
+        type: boolean
-  ARM_CLIENT_SECRET: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
+        required: true
-  ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
-  ARM_TENANT_ID: ${{ secrets.AZURE_E2E_TENANT_ID }}
 
 jobs:
-  generate-measurements-manual:
+  calculate-measurements-on-csp:
+    name: "Calculate Measurements on CSP"
     runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: ["aws", "azure", "gcp"]
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      ARM_CLIENT_ID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
+      ARM_CLIENT_SECRET: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
+      ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
+      ARM_TENANT_ID: ${{ secrets.AZURE_E2E_TENANT_ID }}
     steps:
       - name: Check out repository
         uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # tag=v3.1.0
         with:
           ref: ${{ github.head_ref }}
 
+      - name: Check if image definition from build pipeline exists
+        run: |
+          wget -O /dev/null https://cdn.confidential.cloud/constellation/v1/images/${{ github.event.inputs.osImage }}.json
+        shell: bash
+
       - name: Setup Go environment
         uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f # tag=v3.3.1
         with:
           go-version: "1.19.3"
 
+      - name: Build hack/pcr-reader
+        run: |
+          go build .
+          pwd >> "$GITHUB_PATH"
+        working-directory: hack/pcr-reader
+        shell: bash
+
       - name: Login to Azure
-        if: ${{ github.event.inputs.cloudProvider == 'azure' }}
+        if: matrix.provider == 'azure'
         uses: ./.github/actions/login_azure
         with:
           azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
 
       - name: Create Azure resource group
+        if: matrix.provider == 'azure'
         id: az_resource_group_gen
-        if: ${{ github.event.inputs.cloudProvider == 'azure' }}
         shell: bash
         run: |
           uuid=$(cat /proc/sys/kernel/random/uuid)
           name=e2e-test-${uuid%%-*}
-          az group create --location westus --name "$name" --tags e2e
+          az group create --location northeurope --name "$name" --tags e2e
           echo "res_group_name=$name" >> "$GITHUB_OUTPUT"
 
-      - name: Create Cluster & Generate Measurements
+      - name: Create Cluster in E2E Test environment
-        id: create_and_measure
+        id: create_cluster
-        uses: ./.github/actions/generate_measurements
+        uses: ./.github/actions/e2e_test
         with:
-          cloudProvider: ${{ github.event.inputs.cloudProvider }}
+          workerNodesCount: 1
+          controlNodesCount: 1
+          cloudProvider: ${{ matrix.provider }}
           gcpProject: ${{ secrets.GCP_E2E_PROJECT }}
           gcp_service_account_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
           gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
@@ -72,23 +89,81 @@ jobs:
           azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
           osImage: ${{ github.event.inputs.osImage }}
           isDebugImage: ${{ github.event.inputs.isDebugImage }}
-          cosignPublicKey: ${{ startsWith(github.ref, 'refs/heads/release/v') && secrets.COSIGN_PUBLIC_KEY || secrets.COSIGN_DEV_PUBLIC_KEY }}
+          test: "nop"
-          cosignPrivateKey: ${{ startsWith(github.ref, 'refs/heads/release/v') && secrets.COSIGN_PRIVATE_KEY || secrets.COSIGN_DEV_PRIVATE_KEY }}
+
-          cosignPassword: ${{ startsWith(github.ref, 'refs/heads/release/v') && secrets.COSIGN_PASSWORD || secrets.COSIGN_DEV_PASSWORD }}
+      - name: Fetch PCRs from running cluster
-          awsAccessKeyID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        run: |
-          awsSecretAccessKey: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          KUBECONFIG="${PWD}/constellation-admin.conf" kubectl rollout status ds/verification-service -n kube-system --timeout=3m
-          awsDefaultRegion: ${{ secrets.AWS_DEFAULT_REGION }}
+          CONSTELL_IP=$(jq -r ".ip" constellation-id.json)
-          awsBucketName: ${{ secrets.PUBLIC_BUCKET_NAME }}
+          mkdir -p "${{ github.workspace }}/generated-measurements"
+          pcr-reader -constell-ip "${CONSTELL_IP}" -format json -metadata -csp "${{ matrix.provider }}" -image "${{ github.event.inputs.osImage }}" > "${{ github.workspace }}/generated-measurements/measurements.json"
+          echo "All PCRs of current boot:"
+          cat "${{ github.workspace }}/generated-measurements/measurements.json"
+          case ${CSP} in
+            aws)
+              yq e 'del(.measurements.[1,10,16,17,18,19,20,21,22,23])' -i "${{ github.workspace }}/generated-measurements/measurements.json"
+              yq '.measurements.4.warnOnly = false |
+              .measurements.8.warnOnly = false |
+              .measurements.9.warnOnly = false |
+              .measurements.11.warnOnly = false |
+              .measurements.12.warnOnly = false |
+              .measurements.13.warnOnly = false |
+              .measurements.15.warnOnly = false |
+              .measurements.15.expected = "0000000000000000000000000000000000000000000000000000000000000000"' \
+              -I 0 -o json -i "${{ github.workspace }}/generated-measurements/measurements.json"
+              ;;
+            azure)
+            yq e 'del(.measurements.[0,6,10,16,17,18,19,20,21,22,23])' -I 0 -o json -i "${{ github.workspace }}/generated-measurements/measurements.json"
+            yq '.measurements.4.warnOnly = false |
+            .measurements.8.warnOnly = false |
+            .measurements.9.warnOnly = false |
+            .measurements.11.warnOnly = false |
+            .measurements.12.warnOnly = false |
+            .measurements.13.warnOnly = false |
+            .measurements.15.warnOnly = false |
+            .measurements.15.expected = "0000000000000000000000000000000000000000000000000000000000000000"' \
+            -I 0 -o json -i "${{ github.workspace }}/generated-measurements/measurements.json"
+            ;;
+            gcp)
+            yq e 'del(.measurements.[16,17,18,19,20,21,22,23])' -I 0 -o json -i "${{ github.workspace }}/generated-measurements/measurements.json"
+            yq '.measurements.0.warnOnly = false |
+            .measurements.4.warnOnly = false |
+            .measurements.8.warnOnly = false |
+            .measurements.9.warnOnly = false |
+            .measurements.11.warnOnly = false |
+            .measurements.12.warnOnly = false |
+            .measurements.13.warnOnly = false |
+            .measurements.15.warnOnly = false |
+            .measurements.15.expected = "0000000000000000000000000000000000000000000000000000000000000000"' \
+            -I 0 -o json -i "${{ github.workspace }}/generated-measurements/measurements.json"
+            ;;
+            *)
+              echo "CSP case check failed!"
+              exit 1
+            ;;
+          esac
+          echo "PCRs to be published after removing known variable ones:"
+          cat "${{ github.workspace }}/generated-measurements/measurements.json"
+          mv "${{ github.workspace }}/generated-measurements/measurements.json" "${{ github.workspace }}/generated-measurements/measurements-${{ matrix.provider }}.json"
+        shell: bash
+        env:
+          CSP: ${{ matrix.provider }}
+
+      - name: Upload measurements as artifact
+        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # tag=v3.1.1
+        with:
+          name: measurements-${{ matrix.provider }}.json
+          path: "${{ github.workspace }}/generated-measurements"
 
       - name: Always terminate cluster
         if: always()
         continue-on-error: true
         uses: ./.github/actions/constellation_destroy
         with:
-          kubeconfig: ${{ steps.create_and_measure.outputs.kubeconfig }}
+          kubeconfig: ${{ steps.create_cluster.outputs.kubeconfig }}
 
       - name: Always destroy Azure resource group
-        if: ${{ always() && github.event.inputs.cloudProvider == 'azure' }}
+        if: always() && matrix.provider == 'azure'
         shell: bash
         run: |
           az group delete \
@@ -97,3 +172,143 @@ jobs:
             --force-deletion-types Microsoft.Compute/virtualMachines \
             --no-wait \
             --yes
+
+  validate-measurements:
+    name: "Validate Measurements"
+    needs: [calculate-measurements-on-csp]
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: ["aws", "azure", "gcp"]
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # tag=v3.1.0
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: Setup Go environment
+        uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f # tag=v3.3.1
+        with:
+          go-version: "1.19.3"
+
+      - name: Build hack/pcr-compare
+        run: |
+          go build .
+          pwd >> "$GITHUB_PATH"
+        working-directory: hack/pcr-compare
+        shell: bash
+
+      - name: Download calculated measurements from artifact
+        uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # tag=v3.1.1
+        with:
+          name: measurements-${{ matrix.provider }}.json
+          path: "${{ github.workspace }}/generated-measurements"
+
+      - name: Download expected measurements from build pipeline for image
+        run: |
+          mkdir -p ${{ github.workspace }}/expected-measurements
+          wget -O ${{ github.workspace }}/expected-measurements/measurements.image.json https://cdn.confidential.cloud/constellation/v1/measurements/${{ github.event.inputs.osImage }}/${{ matrix.provider }}/measurements.image.json
+          cat ${{ github.workspace }}/expected-measurements/measurements.image.json
+        shell: bash
+
+      - name: Check if expected measurements == actual measurements from running cluster
+        run: |
+          pcr-compare ${{ github.workspace }}/expected-measurements/measurements.image.json ${{ github.workspace }}/generated-measurements/measurements-${{ matrix.provider }}.json
+        shell: bash
+
+  sign-measurements:
+    name: "Sign Measurements"
+    if: inputs.signMeasurements
+    needs: [calculate-measurements-on-csp, validate-measurements]
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: ["aws", "azure", "gcp"]
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Install Cosign
+        uses: sigstore/cosign-installer@9becc617647dfa20ae7b1151972e9b3a2c338a2b # tag=v2.8.1
+
+      - name: Install Rekor
+        shell: bash
+        run: |
+          curl -sLO https://github.com/sigstore/rekor/releases/download/v0.12.0/rekor-cli-linux-amd64
+          sudo install rekor-cli-linux-amd64 /usr/local/bin/rekor-cli
+          rm rekor-cli-linux-amd64
+
+      - name: Download measurements from artifact
+        uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # tag=v3.0.1
+        with:
+          name: measurements-${{ matrix.provider }}.json
+          path: "${{ github.workspace }}/generated-measurements"
+
+      - name: Sign measurements
+        shell: bash
+        env:
+          COSIGN_PUBLIC_KEY: ${{ startsWith(github.ref, 'refs/heads/release/v') && secrets.COSIGN_PUBLIC_KEY || secrets.COSIGN_DEV_PUBLIC_KEY }}
+          COSIGN_PRIVATE_KEY: ${{ startsWith(github.ref, 'refs/heads/release/v') && secrets.COSIGN_PRIVATE_KEY || secrets.COSIGN_DEV_PRIVATE_KEY }}
+          COSIGN_PASSWORD: ${{ startsWith(github.ref, 'refs/heads/release/v') && secrets.COSIGN_PASSWORD || secrets.COSIGN_DEV_PASSWORD }}
+        run: |
+          echo "${COSIGN_PUBLIC_KEY}" > cosign.pub
+          # Enabling experimental mode also publishes signature to Rekor
+          COSIGN_EXPERIMENTAL=1 cosign sign-blob --key env://COSIGN_PRIVATE_KEY "${{ github.workspace }}/generated-measurements/measurements-${{ matrix.provider }}.json" > "${{ github.workspace }}/generated-measurements/measurements-${{ matrix.provider }}.json.sig"
+          # Verify - As documentation & check
+          # Local Signature (input: artifact, key, signature)
+          cosign verify-blob --key cosign.pub --signature "${{ github.workspace }}/generated-measurements/measurements-${{ matrix.provider }}.json.sig" "${{ github.workspace }}/generated-measurements/measurements-${{ matrix.provider }}.json"
+          # Transparency Log Signature (input: artifact, key)
+          uuid=$(rekor-cli search --artifact "${{ github.workspace }}/generated-measurements/measurements-${{ matrix.provider }}.json" | tail -n 1)
+          sig=$(rekor-cli get --uuid="${uuid}" --format=json | jq -r .Body.HashedRekordObj.signature.content)
+          cosign verify-blob --key cosign.pub --signature <(echo "${sig}") "${{ github.workspace }}/generated-measurements/measurements-${{ matrix.provider }}.json"
+
+      - name: Upload signed measurements as artifact
+        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # tag=v3.1.1
+        with:
+          name: measurements-${{ matrix.provider }}.json.sig
+          path: "${{ github.workspace }}/generated-measurements"
+
+  publish-measurements:
+    name: "Publish Measurements"
+    if: inputs.signMeasurements
+    needs:
+      [calculate-measurements-on-csp, validate-measurements, sign-measurements]
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: ["aws", "azure", "gcp"]
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Download measurements from artifact
+        uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # tag=v3.0.1
+        with:
+          name: measurements-${{ matrix.provider }}.json
+          path: "${{ github.workspace }}/generated-measurements"
+
+      - name: Download signature from artifact
+        uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # tag=v3.0.1
+        with:
+          name: measurements-${{ matrix.provider }}.json.sig
+          path: "${{ github.workspace }}/generated-measurements"
+
+      - name: Login to AWS
+        uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # tag=v1.7.0
+        with:
+          role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline
+          aws-region: eu-central-1
+
+      - name: Upload to S3
+        run: |
+          S3_PATH=s3://cdn-constellation-backend/constellation/v1/measurements/${IMAGE_UID,,}/${{ matrix.provider }}
+          aws s3 cp "${{ github.workspace }}/generated-measurements/measurements-${{ matrix.provider }}.json" "${S3_PATH}/measurements.json"
+          aws s3 cp "${{ github.workspace }}/generated-measurements/measurements-${{ matrix.provider }}.json.sig" "${S3_PATH}/measurements.json.sig"
+        shell: bash
+        env:
+          IMAGE_UID: ${{ inputs.osImage }}
+          PUBLIC_BUCKET_NAME: ${{ secrets.PUBLIC_BUCKET_NAME }}
+          CSP: ${{ matrix.provider }}

hack/go.mod

@@ -38,6 +38,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2 v2.0.0
 	github.com/edgelesssys/constellation/v2 v2.0.0
+	github.com/fatih/color v1.13.0
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/google/go-tpm-tools v0.3.9
 	github.com/google/uuid v1.3.0
@@ -122,7 +123,6 @@ require (
 	github.com/emirpasic/gods v1.12.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
-	github.com/fatih/color v1.13.0 // indirect
 	github.com/go-chi/chi v4.1.2+incompatible // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect

hack/pcr-compare/.gitignore

@@ -0,0 +1 @@
+pcr-compare

hack/pcr-compare/main.go

@@ -0,0 +1,143 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package main
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sort"
+
+	"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
+	"github.com/fatih/color"
+)
+
+func main() {
+	if len(os.Args) < 3 {
+		if len(os.Args) == 0 {
+			fmt.Println("Usage:", "pcr-compare", "<expected-measurements> <actual-measurements>")
+		} else {
+			fmt.Println("Usage:", os.Args[0], "<expected-measurements> <actual-measurements>")
+		}
+		fmt.Println("<expected-measurements> is supposed to be a JSON file from the 'Build OS image' pipeline.")
+		fmt.Println("<actual-measurements> in supposed to be a JSON file with metadata from the PCR reader which is supposed to be verified.")
+		os.Exit(1)
+	}
+
+	parsedExpectedMeasurements, err := parseMeasurements(os.Args[1])
+	if err != nil {
+		panic(err)
+	}
+
+	parsedActualMeasurements, err := parseMeasurements(os.Args[2])
+	if err != nil {
+		panic(err)
+	}
+
+	// Extract the PCR values we care about.
+	strippedActualMeasurements := stripMeasurements(parsedActualMeasurements)
+	strippedExpectedMeasurements := stripMeasurements(parsedExpectedMeasurements)
+
+	// Do the check early.
+	areEqual := strippedExpectedMeasurements.EqualTo(strippedActualMeasurements)
+
+	// Print values and similarities / differences in addition.
+	compareMeasurements(strippedExpectedMeasurements, strippedActualMeasurements)
+
+	if !areEqual {
+		os.Exit(1)
+	}
+}
+
+// parseMeasurements unmarshals a JSON file containing the expected or actual measurements.
+func parseMeasurements(filename string) (measurements.M, error) {
+	fileData, err := os.ReadFile(filename)
+	if err != nil {
+		return measurements.M{}, err
+	}
+
+	// Technically the expected version does not hold metadata, but we can use the same struct as both hold the measurements in `measurements`.
+	// This uses the fallback mechanism of the Measurements unmarshaller which accepts strings without the full struct, defaulting to warnOnly = false.
+	// warnOnly = false is expected for the expected measurements, so that's fine.
+	// We don't verify metadata here, the CLI has to do that.
+	var parsedMeasurements measurements.WithMetadata
+	if err := json.Unmarshal(fileData, &parsedMeasurements); err != nil {
+		return measurements.M{}, err
+	}
+
+	return parsedMeasurements.Measurements, nil
+}
+
+// stripMeasurements extracts only the measurements we want to compare.
+// This excludes PCR 15 since the actual measurements come from an initialized cluster, but the expected measurements are supposed to be from an uninitialized state.
+func stripMeasurements(input measurements.M) measurements.M {
+	toBeChecked := []uint32{4, 8, 9, 11, 12, 13}
+
+	strippedMeasurements := make(measurements.M, len(toBeChecked))
+	for _, pcr := range toBeChecked {
+		if _, ok := input[pcr]; ok {
+			strippedMeasurements[pcr] = input[pcr]
+		}
+	}
+
+	return strippedMeasurements
+}
+
+// compareMeasurements compares the expected PCRs with the actual PCRs.
+func compareMeasurements(expectedMeasurements, actualMeasurements measurements.M) {
+	redPrint := color.New(color.FgRed).SprintFunc()
+	greenPrint := color.New(color.FgGreen).SprintFunc()
+
+	expectedPCRs := getSortedKeysOfMap(expectedMeasurements)
+	for _, pcr := range expectedPCRs {
+		if _, ok := actualMeasurements[pcr]; !ok {
+			color.Magenta("Expected PCR %d not found in the calculated measurements.\n", pcr)
+			continue
+		}
+
+		actualValue := actualMeasurements[pcr]
+		expectedValue := expectedMeasurements[pcr]
+
+		fmt.Printf("Expected PCR %02d: %s (warnOnly: %t)\n", pcr, hex.EncodeToString(expectedValue.Expected[:]), expectedValue.WarnOnly)
+
+		var foundMismatch bool
+		var coloredValue string
+		var coloredWarnOnly string
+		if actualValue.Expected == expectedValue.Expected {
+			coloredValue = greenPrint(hex.EncodeToString(actualValue.Expected[:]))
+		} else {
+			coloredValue = redPrint(hex.EncodeToString(actualValue.Expected[:]))
+			foundMismatch = true
+		}
+
+		if actualValue.WarnOnly == expectedValue.WarnOnly {
+			coloredWarnOnly = greenPrint(fmt.Sprintf("%t", actualValue.WarnOnly))
+		} else {
+			coloredWarnOnly = redPrint(fmt.Sprintf("%t", actualValue.WarnOnly))
+			foundMismatch = true
+		}
+
+		fmt.Printf("Measured PCR %02d: %s (warnOnly: %s)\n", pcr, coloredValue, coloredWarnOnly)
+		if !foundMismatch {
+			color.Green("PCR %02d matches.\n", pcr)
+		} else {
+			color.Red("PCR %02d does not match.\n", pcr)
+		}
+	}
+}
+
+// getSortedKeysOfMap returns the sorted keys of a map to allow the PCR output to be ordered in the output.
+func getSortedKeysOfMap(inputMap measurements.M) []uint32 {
+	keys := make([]uint32, 0, len(inputMap))
+	for singleKey := range inputMap {
+		keys = append(keys, singleKey)
+	}
+	sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
+
+	return keys
+}

hack/pcr-reader/main.go

@@ -17,6 +17,7 @@ import (
 	"net"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
@@ -35,6 +36,10 @@ func main() {
 	format := flag.String("format", "json", "Output format: json, yaml (default json)")
 	quiet := flag.Bool("q", false, "Set to disable output")
 	timeout := flag.Duration("timeout", 2*time.Minute, "Wait this duration for the verification service to become available")
+	metadata := flag.Bool("metadata", false, "Include image metadata (CSP, image UID) for publishing")
+	csp := flag.String("csp", "", "Define CSP for metadata")
+	image := flag.String("image", "", "Define image UID for metadata from which image the PCRs are taken from")
+
 	flag.Parse()
 
 	if *coordIP == "" || *port == "" {
@@ -42,6 +47,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	if *metadata && (*csp == "" || *image == "") {
+		fmt.Println("If you enable metadata, you also need to define a CSP and an image to include from as arguments.")
+		flag.Usage()
+		os.Exit(1)
+	}
+
 	addr := net.JoinHostPort(*coordIP, *port)
 	ctx, cancel := context.WithTimeout(context.Background(), *timeout)
 	defer cancel()
@@ -51,15 +62,28 @@ func main() {
 		log.Fatal(err)
 	}
 
-	measurements, err := validatePCRAttDoc(attDocRaw)
+	pcrs, err := validatePCRAttDoc(attDocRaw)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	if !*quiet {
+	if *quiet {
-		if err := printPCRs(os.Stdout, measurements, *format); err != nil {
+		return
-			log.Fatal(err)
+	}
+
+	if *metadata {
+		outputWithMetadata := measurements.WithMetadata{
+			CSP:          strings.ToLower(*csp),
+			Image:        strings.ToLower(*image),
+			Measurements: pcrs,
 		}
+		err = printPCRsWithMetadata(os.Stdout, outputWithMetadata, *format)
+	} else {
+		err = printPCRs(os.Stdout, pcrs, *format)
+	}
+
+	if err != nil {
+		log.Fatal(err)
 	}
 }
 
@@ -127,6 +151,19 @@ func printPCRs(w io.Writer, pcrs measurements.M, format string) error {
 	}
 }
 
+// printPCRs formats and prints PCRs to the given writer.
+// format can be one of 'json' or 'yaml'. If it doesn't match defaults to 'json'.
+func printPCRsWithMetadata(w io.Writer, outputWithMetadata measurements.WithMetadata, format string) error {
+	switch format {
+	case "json":
+		return printPCRsJSONWithMetadata(w, outputWithMetadata)
+	case "yaml":
+		return printPCRsYAMLWithMetadata(w, outputWithMetadata)
+	default:
+		return printPCRsJSONWithMetadata(w, outputWithMetadata)
+	}
+}
+
 func printPCRsYAML(w io.Writer, pcrs measurements.M) error {
 	pcrYAML, err := yaml.Marshal(pcrs)
 	if err != nil {
@@ -136,6 +173,15 @@ func printPCRsYAML(w io.Writer, pcrs measurements.M) error {
 	return nil
 }
 
+func printPCRsYAMLWithMetadata(w io.Writer, outputWithMetadata measurements.WithMetadata) error {
+	pcrYAML, err := yaml.Marshal(outputWithMetadata)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(w, "%s", string(pcrYAML))
+	return nil
+}
+
 func printPCRsJSON(w io.Writer, pcrs measurements.M) error {
 	pcrJSON, err := json.MarshalIndent(pcrs, "", "  ")
 	if err != nil {
@@ -144,3 +190,12 @@ func printPCRsJSON(w io.Writer, pcrs measurements.M) error {
 	fmt.Fprintf(w, "%s", string(pcrJSON))
 	return nil
 }
+
+func printPCRsJSONWithMetadata(w io.Writer, outputWithMetadata measurements.WithMetadata) error {
+	pcrJSON, err := json.MarshalIndent(outputWithMetadata, "", "  ")
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(w, "%s", string(pcrJSON))
+	return nil
+}

hack/pcr-reader/main_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
+	"gopkg.in/yaml.v3"
 )
 
 func TestMain(m *testing.M) {
@@ -147,3 +148,68 @@ func TestPrintPCRs(t *testing.T) {
 		})
 	}
 }
+
+func TestPrintPCRsWithMetadata(t *testing.T) {
+	testCases := map[string]struct {
+		format string
+		csp    string
+		image  string
+	}{
+		"json": {
+			format: "json",
+			csp:    "azure",
+			image:  "v2.0.0",
+		},
+		"yaml": {
+			csp:    "gcp",
+			image:  "v2.0.0-testimage",
+			format: "yaml",
+		},
+		"empty format": {
+			format: "",
+			csp:    "qemu",
+			image:  "v2.0.0-testimage",
+		},
+		"empty": {},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			pcrs := measurements.M{
+				0: measurements.WithAllBytes(0xAA, true),
+				1: measurements.WithAllBytes(0xBB, true),
+				2: measurements.WithAllBytes(0xCC, true),
+			}
+
+			outputWithMetadata := measurements.WithMetadata{
+				CSP:          tc.csp,
+				Image:        tc.image,
+				Measurements: pcrs,
+			}
+
+			var out bytes.Buffer
+			err := printPCRsWithMetadata(&out, outputWithMetadata, tc.format)
+			assert.NoError(err)
+
+			var unmarshalledOutput measurements.WithMetadata
+			if tc.format == "" || tc.format == "json" {
+				require.NoError(json.Unmarshal(out.Bytes(), &unmarshalledOutput))
+			} else if tc.format == "yaml" {
+				require.NoError(yaml.Unmarshal(out.Bytes(), &unmarshalledOutput))
+			}
+
+			assert.NotNil(unmarshalledOutput.CSP)
+			assert.NotNil(unmarshalledOutput.Image)
+			assert.Equal(tc.csp, unmarshalledOutput.CSP)
+			assert.Equal(tc.image, unmarshalledOutput.Image)
+
+			for idx, pcr := range pcrs {
+				assert.Contains(out.String(), fmt.Sprintf("%d", idx))
+				assert.Contains(out.String(), hex.EncodeToString(pcr.Expected[:]))
+			}
+		})
+	}
+}

internal/attestation/measurements/measurements.go

@@ -38,6 +38,13 @@ const (
 // M are Platform Configuration Register (PCR) values that make up the Measurements.
 type M map[uint32]Measurement
 
+// WithMetadata is a struct supposed to provide CSP & image metadata next to measurements.
+type WithMetadata struct {
+	CSP          string `json:"csp" yaml:"csp"`
+	Image        string `json:"image" yaml:"image"`
+	Measurements M      `json:"measurements" yaml:"measurements"`
+}
+
 // FetchAndVerify fetches measurement and signature files via provided URLs,
 // using client for download. The publicKey is used to verify the measurements.
 // The hash of the fetched measurements is returned.
@@ -146,9 +153,8 @@ type Measurement struct {
 func (m *Measurement) UnmarshalJSON(b []byte) error {
 	var eM encodedMeasurement
 	if err := json.Unmarshal(b, &eM); err != nil {
-		// Unmarshalling failed, Measurement might be in legacy format,
+		// Unmarshalling failed, Measurement might be a simple string instead of Measurement struct.
-		// meaning a simple string instead of Measurement struct.
+		// These values will always be enforced.
-		// TODO: remove with v2.4.0
 		if legacyErr := json.Unmarshal(b, &eM.Expected); legacyErr != nil {
 			return multierr.Append(
 				err,
@@ -176,9 +182,8 @@ func (m Measurement) MarshalJSON() ([]byte, error) {
 func (m *Measurement) UnmarshalYAML(unmarshal func(any) error) error {
 	var eM encodedMeasurement
 	if err := unmarshal(&eM); err != nil {
-		// Unmarshalling failed, Measurement might be in legacy format,
+		// Unmarshalling failed, Measurement might be a simple string instead of Measurement struct.
-		// meaning a simple string instead of Measurement struct.
+		// These values will always be enforced.
-		// TODO: remove with v2.4.0
 		if legacyErr := unmarshal(&eM.Expected); legacyErr != nil {
 			return multierr.Append(
 				err,

rfc/image-discoverability.md

@@ -126,9 +126,9 @@ The format of the image measurements is described in the [secure software distri
 The image measurements are stored in a folder structure in S3 that is organized by CSP and `image version uid`.
 
 ```
-s3://<BUCKET-NAME>/constellation/v1/measurements/<CSP>/<IMAGE-VERSION-UID>/measurements.json
+s3://<BUCKET-NAME>/constellation/v1/measurements/<IMAGE-VERSION-UID>/<CSP>/measurements.json
-s3://<BUCKET-NAME>/constellation/v1/measurements/<CSP>/<IMAGE-VERSION-UID>/measurements.json.sig
+s3://<BUCKET-NAME>/constellation/v1/measurements/<IMAGE-VERSION-UID>/<CSP>/measurements.json.sig
-s3://<BUCKET-NAME>/constellation/v1/measurements/<CSP>/<IMAGE-VERSION-UID>/measurements.image.json
+s3://<BUCKET-NAME>/constellation/v1/measurements/<IMAGE-VERSION-UID>/<CSP>/measurements.image.json
 ```
 
 ## CLI image discovery