e2e: upload TCB versions in verify test

The TCP versions are extracted from the MAA token, that itself is taken
from the verify command output. The configapi is adapted to directly
work on the MAA claims JSON.

Signed-off-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>

.github/actions/e2e_test/action.yml

@@ -63,6 +63,10 @@ inputs:
   githubToken:
     description: "GitHub authorization token"
     required: true
+  cosignPassword:
+    description: "The password for the cosign private key. Used for uploading to the config API"
+  cosignPrivateKey:
+    description: "The cosign private key. Used for uploading to the config API"
   fetchMeasurements:
     description: "Update measurements via the 'constellation config fetch-measurements' command."
     default: "false"
@@ -97,6 +101,15 @@ runs:
           exit 1
         fi
 
+    - name: Validate verify input
+      if: inputs.test == 'verify'
+      shell: bash
+      run: |
+        if [[ "${{ inputs.cosignPassword }}" == '' || "${{ inputs.cosignPrivateKey }}" == '' ]]; then
+          echo "::error::e2e test verify requires cosignPassword and cosignPrivateKey to be set."
+          exit 1
+        fi
+
     - name: Determine build target
       id: determine-build-target
       shell: bash
@@ -290,6 +303,8 @@ runs:
         cloudProvider: ${{ inputs.cloudProvider }}
         osImage: ${{ steps.constellation-create.outputs.osImageUsed }}
         kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
+        cosignPassword: ${{ inputs.cosignPassword }}
+        cosignPrivateKey: ${{ inputs.cosignPrivateKey }}
 
     - name: Run recover test
       if: inputs.test == 'recover'

.github/actions/e2e_verify/action.yml

@@ -11,6 +11,12 @@ inputs:
   kubeconfig:
     description: "The kubeconfig file for the cluster."
     required: true
+  cosignPassword:
+    required: true
+    description: "The password for the cosign private key."
+  cosignPrivateKey:
+    required: true
+    description: "The cosign private key."
 
 runs:
   using: "composite"
@@ -40,6 +46,7 @@ runs:
       env:
         KUBECONFIG: ${{ inputs.kubeconfig }}
       run: |
+        clusterID=$(jq -r ".clusterID" constellation-id.json)
         nodes=$(kubectl get nodes -o json | jq -r ".items[].metadata.name")
 
         for node in $nodes ; do
@@ -52,14 +59,43 @@ runs:
             exit 1
           fi
 
-          echo "Verifying pod ${pod} on node ${node}"
+          echo "Verifying pod ${verificationPod} on node ${node}"
 
           kubectl wait -n kube-system "pod/${verificationPod}" --for=condition=ready --timeout=5m
           kubectl port-forward -n kube-system "pods/${verificationPod}"  9090:9090 &
           forwarderPID=$!
           sleep 5
 
-          constellation verify --cluster-id $(jq -r ".clusterID" constellation-id.json) --force --node-endpoint localhost:9090
+          verifyOut=$(constellation verify --cluster-id "${clusterID}" --force --node-endpoint localhost:9090)
 
           kill $forwarderPID
+
+          if [[ ${{ inputs.cloudProvider }} != "azure" ]]; then
+            continue
+          fi
+
+          echo "Extracting TCB versions for API update"
+          startMAAToken="Microsoft Azure Attestation Token:"
+          endMAAToken="Verification OK"
+          sed -n "/${startMAAToken}/,/${endMAAToken}/ { /${startMAAToken}/d; /${endMAAToken}/d; p }" <<< "${verifyOut}" > "maa-claims-${node}.json"
+        done
+
+    - name: Login to AWS
+      if: github.ref_name == 'main' && inputs.cloudProvider == 'azure'
+      uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
+      with:
+        role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline
+        aws-region: eu-central-1
+
+    - name: Upload extracted TCBs
+      if: github.ref_name == 'main' && inputs.cloudProvider == 'azure'
+      shell: bash
+      env:
+        COSIGN_PASSWORD: ${{ inputs.cosignPassword }}
+        COSIGN_PRIVATE_KEY: ${{ inputs.cosignPrivateKey }}
+      run: |
+        for file in $(ls maa-claims-*.json); do
+          path=$(realpath "${file}")
+          cat "${path}"
+          bazel run //hack/configapi -- --maa-claims-path "${path}"
         done

.github/workflows/e2e-test-daily.yml

@@ -84,6 +84,8 @@ jobs:
           azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           registry: ghcr.io
           githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
+          cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
           fetchMeasurements: ${{ matrix.refStream != 'ref/release/stream/stable/?' }}
 
       - name: Always terminate cluster

.github/workflows/e2e-test-manual.yml

@@ -256,6 +256,8 @@ jobs:
           azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           registry: ghcr.io
           githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
+          cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
           fetchMeasurements: ${{ contains(needs.find-latest-image.outputs.image, '/stream/stable/') }}
 
       - name: Always terminate cluster

.github/workflows/e2e-test-release.yml

@@ -207,7 +207,10 @@ jobs:
           azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
           azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           registry: ghcr.io
+          cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
+          cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
           githubToken: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Always terminate cluster
         if: always()
         uses: ./.github/actions/constellation_destroy

.github/workflows/e2e-test-weekly.yml

@@ -208,6 +208,8 @@ jobs:
           azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           registry: ghcr.io
           githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
+          cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
           fetchMeasurements: ${{ matrix.refStream != 'ref/release/stream/stable/?' }}
           azureSNPEnforcementPolicy: ${{ matrix.azureSNPEnforcementPolicy }}
 
@@ -266,6 +268,7 @@ jobs:
       cloudProvider: ${{ matrix.cloudProvider }}
       nodeCount: '3:2'
       scheduled: ${{ github.event_name == 'schedule' }}
+
   e2e-mini:
     name: Run miniconstellation E2E test
     runs-on: ubuntu-22.04

hack/configapi/BUILD.bazel

@@ -9,7 +9,7 @@ go_library(
 )
 
 go_binary(
-    name = "upload",
+    name = "configapi",
     embed = [":configapi_lib"],
     visibility = ["//visibility:public"],
 )

hack/configapi/cmd/delete.go

@@ -10,8 +10,10 @@ import (
 	"fmt"
 
 	"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
+	"github.com/edgelesssys/constellation/v2/internal/logger"
 	"github.com/edgelesssys/constellation/v2/internal/staticupload"
 	"github.com/spf13/cobra"
+	"go.uber.org/zap"
 )
 
 // newDeleteCmd creates the delete command.
@@ -22,7 +24,7 @@ func newDeleteCmd() *cobra.Command {
 		RunE:  runDelete,
 	}
 	cmd.Flags().StringP("version", "v", "", "Name of the version to delete (without .json suffix)")
-	must(enforceRequiredFlags(cmd, "version"))
+	must(cmd.MarkFlagRequired("version"))
 	return cmd
 }
 
@@ -43,21 +45,22 @@ func (d deleteCmd) delete(cmd *cobra.Command) error {
 }
 
 func runDelete(cmd *cobra.Command, _ []string) error {
+	log := logger.New(logger.PlainLog, zap.DebugLevel).Named("attestationconfigapi")
 	cfg := staticupload.Config{
 		Bucket: awsBucket,
 		Region: awsRegion,
 	}
-	repo, closefn, err := attestationconfigapi.NewClient(cmd.Context(), cfg, []byte(cosignPwd), []byte(privateKey), false, log())
+	client, close, err := attestationconfigapi.NewClient(cmd.Context(), cfg, []byte(cosignPwd), []byte(privateKey), false, log)
 	if err != nil {
 		return fmt.Errorf("create attestation client: %w", err)
 	}
 	defer func() {
-		if err := closefn(cmd.Context()); err != nil {
+		if err := close(cmd.Context()); err != nil {
 			cmd.Printf("close client: %s\n", err.Error())
 		}
 	}()
 	deleteCmd := deleteCmd{
-		attestationClient: repo,
+		attestationClient: client,
 	}
 	return deleteCmd.delete(cmd)
 }

hack/configapi/cmd/root.go

@@ -10,7 +10,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"reflect"
 	"time"
 
 	"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
@@ -24,16 +23,12 @@ import (
 const (
 	awsRegion           = "eu-central-1"
 	awsBucket           = "cdn-constellation-backend"
-	invalidDefault      = 0
-	envAwsKeyID         = "AWS_ACCESS_KEY_ID"
-	envAwsKey           = "AWS_ACCESS_KEY"
 	envCosignPwd        = "COSIGN_PASSWORD"
 	envCosignPrivateKey = "COSIGN_PRIVATE_KEY"
 )
 
 var (
-	versionFilePath string
+	maaFilePath string
-	force           bool
 	// Cosign credentials.
 	cosignPwd  string
 	privateKey string
@@ -47,17 +42,20 @@ func Execute() error {
 // newRootCmd creates the root command.
 func newRootCmd() *cobra.Command {
 	rootCmd := &cobra.Command{
-		Use:   "COSIGN_PASSWORD=$CPWD COSIGN_PRIVATE_KEY=$CKEY AWS_ACCESS_KEY_ID=$ID AWS_ACCESS_KEY=$KEY upload --version-file $FILE",
+		Use:   "COSIGN_PASSWORD=$CPWD COSIGN_PRIVATE_KEY=$CKEY upload --version-file $FILE",
 		Short: "Upload a set of versions specific to the azure-sev-snp attestation variant to the config api.",
 
-		Long:    fmt.Sprintf("Upload a set of versions specific to the azure-sev-snp attestation variant to the config api. Please authenticate with AWS through your preferred method (e.g. environment variables, CLI) to be able to upload to S3. Set the %s and %s environment variables to authenticate with cosign.", envCosignPrivateKey, envCosignPwd),
+		Long: fmt.Sprintf("Upload a set of versions specific to the azure-sev-snp attestation variant to the config api."+
+			"Please authenticate with AWS through your preferred method (e.g. environment variables, CLI)"+
+			"to be able to upload to S3. Set the %s and %s environment variables to authenticate with cosign.",
+			envCosignPrivateKey, envCosignPwd,
+		),
 		PreRunE: envCheck,
 		RunE:    runCmd,
 	}
-	rootCmd.Flags().StringVarP(&versionFilePath, "version-file", "f", "", "File path to the version json file.")
+	rootCmd.Flags().StringVarP(&maaFilePath, "maa-claims-path", "t", "", "File path to a json file containing the MAA claims.")
-	rootCmd.Flags().BoolVar(&force, "force", false, "force to upload version regardless of comparison to latest API value.")
+	rootCmd.Flags().StringP("upload-date", "d", "", "upload a version with this date as version name.")
-	rootCmd.Flags().StringP("upload-date", "d", "", "upload a version with this date as version name. Setting it implies --force.")
+	must(rootCmd.MarkFlagRequired("maa-claims-path"))
-	must(enforceRequiredFlags(rootCmd, "version-file"))
 	rootCmd.AddCommand(newDeleteCmd())
 	return rootCmd
 }
@@ -73,113 +71,103 @@ func envCheck(_ *cobra.Command, _ []string) error {
 
 func runCmd(cmd *cobra.Command, _ []string) error {
 	ctx := cmd.Context()
+	log := logger.New(logger.PlainLog, zap.DebugLevel).Named("attestationconfigapi")
 	cfg := staticupload.Config{
 		Bucket: awsBucket,
 		Region: awsRegion,
 	}
-	versionBytes, err := os.ReadFile(versionFilePath)
+	maaClaimsBytes, err := os.ReadFile(maaFilePath)
 	if err != nil {
-		return fmt.Errorf("reading version file: %w", err)
+		return fmt.Errorf("reading MAA claims file: %w", err)
 	}
-	var inputVersion attestationconfigapi.AzureSEVSNPVersion
+	var maaTCB maaTokenTCBClaims
-	if err = json.Unmarshal(versionBytes, &inputVersion); err != nil {
+	if err = json.Unmarshal(maaClaimsBytes, &maaTCB); err != nil {
-		return fmt.Errorf("unmarshalling version file: %w", err)
+		return fmt.Errorf("unmarshalling MAA claims file: %w", err)
 	}
+	inputVersion := maaTCB.ToAzureSEVSNPVersion()
 
 	dateStr, err := cmd.Flags().GetString("upload-date")
 	if err != nil {
 		return fmt.Errorf("getting upload date: %w", err)
 	}
-	var uploadDate time.Time
+	uploadDate := time.Now()
 	if dateStr != "" {
 		uploadDate, err = time.Parse(attestationconfigapi.VersionFormat, dateStr)
 		if err != nil {
 			return fmt.Errorf("parsing date: %w", err)
 		}
-	} else {
-		uploadDate = time.Now()
-		force = true
 	}
 
-	doUpload := false
+	latestAPIVersion, err := attestationconfigapi.NewFetcher().FetchAzureSEVSNPVersionLatest(ctx, uploadDate)
-	if !force {
+	if err != nil {
-		latestAPIVersion, err := attestationconfigapi.NewFetcher().FetchAzureSEVSNPVersionLatest(ctx, time.Now())
+		return fmt.Errorf("fetching latest version: %w", err)
-		if err != nil {
-			return fmt.Errorf("fetching latest version: %w", err)
-		}
-
-		isNewer, err := isInputNewerThanLatestAPI(inputVersion, latestAPIVersion.AzureSEVSNPVersion)
-		if err != nil {
-			return fmt.Errorf("comparing versions: %w", err)
-		}
-		cmd.Print(versionComparisonInformation(isNewer, inputVersion, latestAPIVersion.AzureSEVSNPVersion))
-		doUpload = isNewer
-	} else {
-		doUpload = true
-		cmd.Println("Forcing upload of new version")
 	}
 
-	if doUpload {
+	isNewer, err := isInputNewerThanLatestAPI(inputVersion, latestAPIVersion.AzureSEVSNPVersion)
-		sut, sutClose, err := attestationconfigapi.NewClient(ctx, cfg, []byte(cosignPwd), []byte(privateKey), false, log())
+	if err != nil {
-		defer func() {
+		return fmt.Errorf("comparing versions: %w", err)
-			if err := sutClose(ctx); err != nil {
-				cmd.Printf("closing repo: %v\n", err)
-			}
-		}()
-		if err != nil {
-			return fmt.Errorf("creating repo: %w", err)
-		}
-
-		if err := sut.UploadAzureSEVSNP(ctx, inputVersion, uploadDate); err != nil {
-			return fmt.Errorf("uploading version: %w", err)
-		}
-		cmd.Printf("Successfully uploaded new Azure SEV-SNP version: %+v\n", inputVersion)
 	}
+	if !isNewer {
+		fmt.Printf("Input version: %+v is not newer than latest API version: %+v\n", inputVersion, latestAPIVersion)
+		return nil
+	}
+	fmt.Printf("Input version: %+v is newer than latest API version: %+v\n", inputVersion, latestAPIVersion)
+
+	client, stop, err := attestationconfigapi.NewClient(ctx, cfg, []byte(cosignPwd), []byte(privateKey), false, log)
+	defer func() {
+		if err := stop(ctx); err != nil {
+			cmd.Printf("stopping client: %v\n", err)
+		}
+	}()
+	if err != nil {
+		return fmt.Errorf("creating client: %w", err)
+	}
+
+	if err := client.UploadAzureSEVSNP(ctx, inputVersion, uploadDate); err != nil {
+		return fmt.Errorf("uploading version: %w", err)
+	}
+
+	cmd.Printf("Successfully uploaded new Azure SEV-SNP version: %+v\n", inputVersion)
 	return nil
 }
 
-func versionComparisonInformation(isNewer bool, inputVersion attestationconfigapi.AzureSEVSNPVersion, latestAPIVersion attestationconfigapi.AzureSEVSNPVersion) string {
+// maaTokenTCBClaims describes the TCB information in a MAA token.
-	if isNewer {
+type maaTokenTCBClaims struct {
-		return fmt.Sprintf("Input version: %+v is newer than latest API version: %+v\n", inputVersion, latestAPIVersion)
+	TEESvn        uint8 `json:"x-ms-sevsnpvm-tee-svn"`
+	SNPFwSvn      uint8 `json:"x-ms-sevsnpvm-snpfw-svn"`
+	MicrocodeSvn  uint8 `json:"x-ms-sevsnpvm-microcode-svn"`
+	BootloaderSvn uint8 `json:"x-ms-sevsnpvm-bootloader-svn"`
+}
+
+func (c maaTokenTCBClaims) ToAzureSEVSNPVersion() attestationconfigapi.AzureSEVSNPVersion {
+	return attestationconfigapi.AzureSEVSNPVersion{
+		TEE:        c.TEESvn,
+		SNP:        c.SNPFwSvn,
+		Microcode:  c.MicrocodeSvn,
+		Bootloader: c.BootloaderSvn,
 	}
-	return fmt.Sprintf("Input version: %+v is not newer than latest API version: %+v\n", inputVersion, latestAPIVersion)
 }
 
 // isInputNewerThanLatestAPI compares all version fields with the latest API version and returns true if any input field is newer.
 func isInputNewerThanLatestAPI(input, latest attestationconfigapi.AzureSEVSNPVersion) (bool, error) {
-	inputValues := reflect.ValueOf(input)
+	if input == latest {
-	latestValues := reflect.ValueOf(latest)
+		return false, nil
-	fields := reflect.TypeOf(input)
-	num := fields.NumField()
-	// validate that no input field is smaller than latest
-	for i := 0; i < num; i++ {
-		field := fields.Field(i)
-		inputValue := inputValues.Field(i).Uint()
-		latestValue := latestValues.Field(i).Uint()
-		if inputValue < latestValue {
-			return false, fmt.Errorf("input %s version: %d is older than latest API version: %d", field.Name, inputValue, latestValue)
-		} else if inputValue > latestValue {
-			return true, nil
-		}
 	}
-	// check if any input field is greater than latest
-	for i := 0; i < num; i++ {
-		inputValue := inputValues.Field(i).Uint()
-		latestValue := latestValues.Field(i).Uint()
-		if inputValue > latestValue {
-			return true, nil
-		}
-	}
-	return false, nil
-}
 
-func enforceRequiredFlags(cmd *cobra.Command, flags ...string) error {
+	if input.TEE < latest.TEE {
-	for _, flag := range flags {
+		return false, fmt.Errorf("input TEE version: %d is older than latest API version: %d", input.TEE, latest.TEE)
-		if err := cmd.MarkFlagRequired(flag); err != nil {
-			return err
-		}
 	}
-	return nil
+	if input.SNP < latest.SNP {
+		return false, fmt.Errorf("input SNP version: %d is older than latest API version: %d", input.SNP, latest.SNP)
+	}
+	if input.Microcode < latest.Microcode {
+		return false, fmt.Errorf("input Microcode version: %d is older than latest API version: %d", input.Microcode, latest.Microcode)
+	}
+	if input.Bootloader < latest.Bootloader {
+		return false, fmt.Errorf("input Bootloader version: %d is older than latest API version: %d", input.Bootloader, latest.Bootloader)
+	}
+
+	return true, nil
 }
 
 func must(err error) {
@@ -187,7 +175,3 @@ func must(err error) {
 		panic(err)
 	}
 }
-
-func log() *logger.Logger {
-	return logger.New(logger.PlainLog, zap.DebugLevel).Named("attestationconfigapi")
-}