e2e: upload TCB versions in verify test

The TCP versions are extracted from the MAA token, that itself is taken
from the verify command output. The configapi is adapted to directly
work on the MAA claims JSON.

Signed-off-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>

.github/actions/e2e_test/action.yml

@@ -63,6 +63,10 @@ inputs:
   githubToken:
     description: "GitHub authorization token"
     required: true
+  cosignPassword:
+    description: "The password for the cosign private key. Used for uploading to the config API"
+  cosignPrivateKey:
+    description: "The cosign private key. Used for uploading to the config API"
   fetchMeasurements:
     description: "Update measurements via the 'constellation config fetch-measurements' command."
     default: "false"
@@ -97,6 +101,15 @@ runs:
           exit 1
         fi
 
+    - name: Validate verify input
+      if: inputs.test == 'verify'
+      shell: bash
+      run: |
+        if [[ "${{ inputs.cosignPassword }}" == '' || "${{ inputs.cosignPrivateKey }}" == '' ]]; then
+          echo "::error::e2e test verify requires cosignPassword and cosignPrivateKey to be set."
+          exit 1
+        fi
+
     - name: Determine build target
       id: determine-build-target
       shell: bash
@@ -290,6 +303,8 @@ runs:
         cloudProvider: ${{ inputs.cloudProvider }}
         osImage: ${{ steps.constellation-create.outputs.osImageUsed }}
         kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
+        cosignPassword: ${{ inputs.cosignPassword }}
+        cosignPrivateKey: ${{ inputs.cosignPrivateKey }}
 
     - name: Run recover test
       if: inputs.test == 'recover'

.github/actions/e2e_verify/action.yml

@@ -11,6 +11,12 @@ inputs:
   kubeconfig:
     description: "The kubeconfig file for the cluster."
     required: true
+  cosignPassword:
+    required: true
+    description: "The password for the cosign private key."
+  cosignPrivateKey:
+    required: true
+    description: "The cosign private key."
 
 runs:
   using: "composite"
@@ -40,6 +46,7 @@ runs:
       env:
         KUBECONFIG: ${{ inputs.kubeconfig }}
       run: |
+        clusterID=$(jq -r ".clusterID" constellation-id.json)
         nodes=$(kubectl get nodes -o json | jq -r ".items[].metadata.name")
 
         for node in $nodes ; do
@@ -52,14 +59,43 @@ runs:
             exit 1
           fi
 
-          echo "Verifying pod ${pod} on node ${node}"
+          echo "Verifying pod ${verificationPod} on node ${node}"
 
           kubectl wait -n kube-system "pod/${verificationPod}" --for=condition=ready --timeout=5m
           kubectl port-forward -n kube-system "pods/${verificationPod}"  9090:9090 &
           forwarderPID=$!
           sleep 5
 
-          constellation verify --cluster-id $(jq -r ".clusterID" constellation-id.json) --force --node-endpoint localhost:9090
+          verifyOut=$(constellation verify --cluster-id "${clusterID}" --force --node-endpoint localhost:9090)
 
           kill $forwarderPID
+
+          if [[ ${{ inputs.cloudProvider }} != "azure" ]]; then
+            continue
+          fi
+
+          echo "Extracting TCB versions for API update"
+          startMAAToken="Microsoft Azure Attestation Token:"
+          endMAAToken="Verification OK"
+          sed -n "/${startMAAToken}/,/${endMAAToken}/ { /${startMAAToken}/d; /${endMAAToken}/d; p }" <<< "${verifyOut}" > "maa-claims-${node}.json"
+        done
+
+    - name: Login to AWS
+      if: github.ref_name == 'main' && inputs.cloudProvider == 'azure'
+      uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
+      with:
+        role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline
+        aws-region: eu-central-1
+
+    - name: Upload extracted TCBs
+      if: github.ref_name == 'main' && inputs.cloudProvider == 'azure'
+      shell: bash
+      env:
+        COSIGN_PASSWORD: ${{ inputs.cosignPassword }}
+        COSIGN_PRIVATE_KEY: ${{ inputs.cosignPrivateKey }}
+      run: |
+        for file in $(ls maa-claims-*.json); do
+          path=$(realpath "${file}")
+          cat "${path}"
+          bazel run //hack/configapi -- --maa-claims-path "${path}"
         done

.github/workflows/e2e-test-daily.yml

@@ -84,6 +84,8 @@ jobs:
           azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           registry: ghcr.io
           githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
+          cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
           fetchMeasurements: ${{ matrix.refStream != 'ref/release/stream/stable/?' }}
 
       - name: Always terminate cluster

.github/workflows/e2e-test-manual.yml

@@ -256,6 +256,8 @@ jobs:
           azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           registry: ghcr.io
           githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
+          cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
           fetchMeasurements: ${{ contains(needs.find-latest-image.outputs.image, '/stream/stable/') }}
 
       - name: Always terminate cluster

.github/workflows/e2e-test-release.yml

@@ -207,7 +207,10 @@ jobs:
           azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
           azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           registry: ghcr.io
+          cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
+          cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
           githubToken: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Always terminate cluster
         if: always()
         uses: ./.github/actions/constellation_destroy

.github/workflows/e2e-test-weekly.yml

@@ -208,6 +208,8 @@ jobs:
           azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           registry: ghcr.io
           githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
+          cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
           fetchMeasurements: ${{ matrix.refStream != 'ref/release/stream/stable/?' }}
           azureSNPEnforcementPolicy: ${{ matrix.azureSNPEnforcementPolicy }}
 
@@ -266,6 +268,7 @@ jobs:
       cloudProvider: ${{ matrix.cloudProvider }}
       nodeCount: '3:2'
       scheduled: ${{ github.event_name == 'schedule' }}
+
   e2e-mini:
     name: Run miniconstellation E2E test
     runs-on: ubuntu-22.04

hack/configapi/BUILD.bazel

@@ -9,7 +9,7 @@ go_library(
 )
 
 go_binary(
-    name = "upload",
+    name = "configapi",
     embed = [":configapi_lib"],
     visibility = ["//visibility:public"],
 )

hack/configapi/cmd/delete.go

@@ -10,8 +10,10 @@ import (
 	"fmt"
 
 	"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
+	"github.com/edgelesssys/constellation/v2/internal/logger"
 	"github.com/edgelesssys/constellation/v2/internal/staticupload"
 	"github.com/spf13/cobra"
+	"go.uber.org/zap"
 )
 
 // newDeleteCmd creates the delete command.
@@ -22,7 +24,7 @@ func newDeleteCmd() *cobra.Command {
 		RunE:  runDelete,
 	}
 	cmd.Flags().StringP("version", "v", "", "Name of the version to delete (without .json suffix)")
-	must(enforceRequiredFlags(cmd, "version"))
+	must(cmd.MarkFlagRequired("version"))
 	return cmd
 }
 
@@ -43,21 +45,22 @@ func (d deleteCmd) delete(cmd *cobra.Command) error {
 }
 
 func runDelete(cmd *cobra.Command, _ []string) error {
+	log := logger.New(logger.PlainLog, zap.DebugLevel).Named("attestationconfigapi")
 	cfg := staticupload.Config{
 		Bucket: awsBucket,
 		Region: awsRegion,
 	}
-	repo, closefn, err := attestationconfigapi.NewClient(cmd.Context(), cfg, []byte(cosignPwd), []byte(privateKey), false, log())
+	client, close, err := attestationconfigapi.NewClient(cmd.Context(), cfg, []byte(cosignPwd), []byte(privateKey), false, log)
 	if err != nil {
 		return fmt.Errorf("create attestation client: %w", err)
 	}
 	defer func() {
-		if err := closefn(cmd.Context()); err != nil {
+		if err := close(cmd.Context()); err != nil {
 			cmd.Printf("close client: %s\n", err.Error())
 		}
 	}()
 	deleteCmd := deleteCmd{
-		attestationClient: repo,
+		attestationClient: client,
 	}
 	return deleteCmd.delete(cmd)
 }

hack/configapi/cmd/root.go

@@ -10,7 +10,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"reflect"
 	"time"
 
 	"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
@@ -24,16 +23,12 @@ import (
 const (
 	awsRegion           = "eu-central-1"
 	awsBucket           = "cdn-constellation-backend"
-	invalidDefault      = 0
-	envAwsKeyID         = "AWS_ACCESS_KEY_ID"
-	envAwsKey           = "AWS_ACCESS_KEY"
 	envCosignPwd        = "COSIGN_PASSWORD"
 	envCosignPrivateKey = "COSIGN_PRIVATE_KEY"
 )
 
 var (
-	versionFilePath string
-	force           bool
+	maaFilePath string
 	// Cosign credentials.
 	cosignPwd  string
 	privateKey string
@@ -47,17 +42,20 @@ func Execute() error {
 // newRootCmd creates the root command.
 func newRootCmd() *cobra.Command {
 	rootCmd := &cobra.Command{
-		Use:   "COSIGN_PASSWORD=$CPWD COSIGN_PRIVATE_KEY=$CKEY AWS_ACCESS_KEY_ID=$ID AWS_ACCESS_KEY=$KEY upload --version-file $FILE",
+		Use:   "COSIGN_PASSWORD=$CPWD COSIGN_PRIVATE_KEY=$CKEY upload --version-file $FILE",
 		Short: "Upload a set of versions specific to the azure-sev-snp attestation variant to the config api.",
 
-		Long:    fmt.Sprintf("Upload a set of versions specific to the azure-sev-snp attestation variant to the config api. Please authenticate with AWS through your preferred method (e.g. environment variables, CLI) to be able to upload to S3. Set the %s and %s environment variables to authenticate with cosign.", envCosignPrivateKey, envCosignPwd),
+		Long: fmt.Sprintf("Upload a set of versions specific to the azure-sev-snp attestation variant to the config api."+
+			"Please authenticate with AWS through your preferred method (e.g. environment variables, CLI)"+
+			"to be able to upload to S3. Set the %s and %s environment variables to authenticate with cosign.",
+			envCosignPrivateKey, envCosignPwd,
+		),
 		PreRunE: envCheck,
 		RunE:    runCmd,
 	}
-	rootCmd.Flags().StringVarP(&versionFilePath, "version-file", "f", "", "File path to the version json file.")
-	rootCmd.Flags().BoolVar(&force, "force", false, "force to upload version regardless of comparison to latest API value.")
-	rootCmd.Flags().StringP("upload-date", "d", "", "upload a version with this date as version name. Setting it implies --force.")
-	must(enforceRequiredFlags(rootCmd, "version-file"))
+	rootCmd.Flags().StringVarP(&maaFilePath, "maa-claims-path", "t", "", "File path to a json file containing the MAA claims.")
+	rootCmd.Flags().StringP("upload-date", "d", "", "upload a version with this date as version name.")
+	must(rootCmd.MarkFlagRequired("maa-claims-path"))
 	rootCmd.AddCommand(newDeleteCmd())
 	return rootCmd
 }
@@ -73,113 +71,103 @@ func envCheck(_ *cobra.Command, _ []string) error {
 
 func runCmd(cmd *cobra.Command, _ []string) error {
 	ctx := cmd.Context()
+	log := logger.New(logger.PlainLog, zap.DebugLevel).Named("attestationconfigapi")
 	cfg := staticupload.Config{
 		Bucket: awsBucket,
 		Region: awsRegion,
 	}
-	versionBytes, err := os.ReadFile(versionFilePath)
+	maaClaimsBytes, err := os.ReadFile(maaFilePath)
 	if err != nil {
-		return fmt.Errorf("reading version file: %w", err)
+		return fmt.Errorf("reading MAA claims file: %w", err)
 	}
-	var inputVersion attestationconfigapi.AzureSEVSNPVersion
-	if err = json.Unmarshal(versionBytes, &inputVersion); err != nil {
-		return fmt.Errorf("unmarshalling version file: %w", err)
+	var maaTCB maaTokenTCBClaims
+	if err = json.Unmarshal(maaClaimsBytes, &maaTCB); err != nil {
+		return fmt.Errorf("unmarshalling MAA claims file: %w", err)
 	}
+	inputVersion := maaTCB.ToAzureSEVSNPVersion()
 
 	dateStr, err := cmd.Flags().GetString("upload-date")
 	if err != nil {
 		return fmt.Errorf("getting upload date: %w", err)
 	}
-	var uploadDate time.Time
+	uploadDate := time.Now()
 	if dateStr != "" {
 		uploadDate, err = time.Parse(attestationconfigapi.VersionFormat, dateStr)
 		if err != nil {
 			return fmt.Errorf("parsing date: %w", err)
 		}
-	} else {
-		uploadDate = time.Now()
-		force = true
 	}
 
-	doUpload := false
-	if !force {
-		latestAPIVersion, err := attestationconfigapi.NewFetcher().FetchAzureSEVSNPVersionLatest(ctx, time.Now())
-		if err != nil {
-			return fmt.Errorf("fetching latest version: %w", err)
-		}
-
-		isNewer, err := isInputNewerThanLatestAPI(inputVersion, latestAPIVersion.AzureSEVSNPVersion)
-		if err != nil {
-			return fmt.Errorf("comparing versions: %w", err)
-		}
-		cmd.Print(versionComparisonInformation(isNewer, inputVersion, latestAPIVersion.AzureSEVSNPVersion))
-		doUpload = isNewer
-	} else {
-		doUpload = true
-		cmd.Println("Forcing upload of new version")
+	latestAPIVersion, err := attestationconfigapi.NewFetcher().FetchAzureSEVSNPVersionLatest(ctx, uploadDate)
+	if err != nil {
+		return fmt.Errorf("fetching latest version: %w", err)
 	}
 
-	if doUpload {
-		sut, sutClose, err := attestationconfigapi.NewClient(ctx, cfg, []byte(cosignPwd), []byte(privateKey), false, log())
-		defer func() {
-			if err := sutClose(ctx); err != nil {
-				cmd.Printf("closing repo: %v\n", err)
-			}
-		}()
-		if err != nil {
-			return fmt.Errorf("creating repo: %w", err)
-		}
-
-		if err := sut.UploadAzureSEVSNP(ctx, inputVersion, uploadDate); err != nil {
-			return fmt.Errorf("uploading version: %w", err)
-		}
-		cmd.Printf("Successfully uploaded new Azure SEV-SNP version: %+v\n", inputVersion)
+	isNewer, err := isInputNewerThanLatestAPI(inputVersion, latestAPIVersion.AzureSEVSNPVersion)
+	if err != nil {
+		return fmt.Errorf("comparing versions: %w", err)
 	}
+	if !isNewer {
+		fmt.Printf("Input version: %+v is not newer than latest API version: %+v\n", inputVersion, latestAPIVersion)
+		return nil
+	}
+	fmt.Printf("Input version: %+v is newer than latest API version: %+v\n", inputVersion, latestAPIVersion)
+
+	client, stop, err := attestationconfigapi.NewClient(ctx, cfg, []byte(cosignPwd), []byte(privateKey), false, log)
+	defer func() {
+		if err := stop(ctx); err != nil {
+			cmd.Printf("stopping client: %v\n", err)
+		}
+	}()
+	if err != nil {
+		return fmt.Errorf("creating client: %w", err)
+	}
+
+	if err := client.UploadAzureSEVSNP(ctx, inputVersion, uploadDate); err != nil {
+		return fmt.Errorf("uploading version: %w", err)
+	}
+
+	cmd.Printf("Successfully uploaded new Azure SEV-SNP version: %+v\n", inputVersion)
 	return nil
 }
 
-func versionComparisonInformation(isNewer bool, inputVersion attestationconfigapi.AzureSEVSNPVersion, latestAPIVersion attestationconfigapi.AzureSEVSNPVersion) string {
-	if isNewer {
-		return fmt.Sprintf("Input version: %+v is newer than latest API version: %+v\n", inputVersion, latestAPIVersion)
+// maaTokenTCBClaims describes the TCB information in a MAA token.
+type maaTokenTCBClaims struct {
+	TEESvn        uint8 `json:"x-ms-sevsnpvm-tee-svn"`
+	SNPFwSvn      uint8 `json:"x-ms-sevsnpvm-snpfw-svn"`
+	MicrocodeSvn  uint8 `json:"x-ms-sevsnpvm-microcode-svn"`
+	BootloaderSvn uint8 `json:"x-ms-sevsnpvm-bootloader-svn"`
+}
+
+func (c maaTokenTCBClaims) ToAzureSEVSNPVersion() attestationconfigapi.AzureSEVSNPVersion {
+	return attestationconfigapi.AzureSEVSNPVersion{
+		TEE:        c.TEESvn,
+		SNP:        c.SNPFwSvn,
+		Microcode:  c.MicrocodeSvn,
+		Bootloader: c.BootloaderSvn,
 	}
-	return fmt.Sprintf("Input version: %+v is not newer than latest API version: %+v\n", inputVersion, latestAPIVersion)
 }
 
 // isInputNewerThanLatestAPI compares all version fields with the latest API version and returns true if any input field is newer.
 func isInputNewerThanLatestAPI(input, latest attestationconfigapi.AzureSEVSNPVersion) (bool, error) {
-	inputValues := reflect.ValueOf(input)
-	latestValues := reflect.ValueOf(latest)
-	fields := reflect.TypeOf(input)
-	num := fields.NumField()
-	// validate that no input field is smaller than latest
-	for i := 0; i < num; i++ {
-		field := fields.Field(i)
-		inputValue := inputValues.Field(i).Uint()
-		latestValue := latestValues.Field(i).Uint()
-		if inputValue < latestValue {
-			return false, fmt.Errorf("input %s version: %d is older than latest API version: %d", field.Name, inputValue, latestValue)
-		} else if inputValue > latestValue {
-			return true, nil
-		}
+	if input == latest {
+		return false, nil
 	}
-	// check if any input field is greater than latest
-	for i := 0; i < num; i++ {
-		inputValue := inputValues.Field(i).Uint()
-		latestValue := latestValues.Field(i).Uint()
-		if inputValue > latestValue {
-			return true, nil
-		}
-	}
-	return false, nil
-}
 
-func enforceRequiredFlags(cmd *cobra.Command, flags ...string) error {
-	for _, flag := range flags {
-		if err := cmd.MarkFlagRequired(flag); err != nil {
-			return err
-		}
+	if input.TEE < latest.TEE {
+		return false, fmt.Errorf("input TEE version: %d is older than latest API version: %d", input.TEE, latest.TEE)
 	}
-	return nil
+	if input.SNP < latest.SNP {
+		return false, fmt.Errorf("input SNP version: %d is older than latest API version: %d", input.SNP, latest.SNP)
+	}
+	if input.Microcode < latest.Microcode {
+		return false, fmt.Errorf("input Microcode version: %d is older than latest API version: %d", input.Microcode, latest.Microcode)
+	}
+	if input.Bootloader < latest.Bootloader {
+		return false, fmt.Errorf("input Bootloader version: %d is older than latest API version: %d", input.Bootloader, latest.Bootloader)
+	}
+
+	return true, nil
 }
 
 func must(err error) {
@@ -187,7 +175,3 @@ func must(err error) {
 		panic(err)
 	}
 }
-
-func log() *logger.Logger {
-	return logger.New(logger.PlainLog, zap.DebugLevel).Named("attestationconfigapi")
-}