Configapi: pipeline to run e2e test for CLI

Co-authored-by: Paul Meyer <pm@edgeless.systems>

.github/actions/e2e_attestationconfigapi/action.yml

@@ -0,0 +1,36 @@
+name: E2E Attestationconfig API Test
+description: "Test the attestationconfig CLI is functional."
+
+inputs:
+  buildBuddyApiKey:
+    description: "BuildBuddy API key for caching Bazel artifacts"
+    required: true
+  cosignPrivateKey:
+    description: "Cosign private key"
+    required: true
+  cosignPassword:
+    description: "Password for Cosign private key"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup bazel
+      uses: ./.github/actions/setup_bazel
+      with:
+        useCache: "true"
+        buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }}
+
+    - name: Login to AWS
+      uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
+      with:
+        role-to-assume: arn:aws:iam::795746500882:role/GithubTestResourceAPI
+        aws-region: eu-west-1
+
+    - name: Run attestationconfig API E2E
+      shell: bash
+      env:
+        COSIGN_PRIVATE_KEY: ${{ inputs.cosignPrivateKey }}
+        COSIGN_PASSWORD: ${{ inputs.cosignPassword }}
+      run: |
+        bazel run //hack/configapi:configapi_e2e_test

.github/workflows/e2e-attestationconfigapi.yml

@@ -0,0 +1,36 @@
+name: e2e test attestationconfig API
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - "release/**"
+    paths:
+      - "internal/api/**"
+      - ".github/workflows/e2e-attestationconfigapi.yml"
+  pull_request:
+    paths:
+      - "internal/api/**"
+      - ".github/workflows/e2e-attestationconfigapi.yml"
+
+jobs:
+  e2e-api:
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        with:
+          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
+
+      - name: Run Attestationconfig API E2E
+        uses: ./.github/actions/e2e_attestationconfigapi
+        with:
+          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+          cosignPrivateKey: ${{ secrets.COSIGN_DEV_PRIVATE_KEY }}
+          cosignPassword: ${{ secrets.COSIGN_DEV_PASSWORD }}

hack/configapi/BUILD.bazel

@@ -1,5 +1,6 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
 load("//bazel/go:go_test.bzl", "go_test")
+load("//bazel/sh:def.bzl", "sh_template")
 
 go_library(
     name = "configapi_lib",
@@ -11,6 +12,7 @@ go_library(
     visibility = ["//visibility:private"],
     deps = [
         "//internal/api/attestationconfigapi",
+        "//internal/constants",
         "//internal/logger",
         "//internal/staticupload",
         "@com_github_spf13_cobra//:cobra",
@@ -37,3 +39,12 @@ go_test(
         "@com_github_stretchr_testify//require",
     ],
 )
+
+sh_template(
+    name = "configapi_e2e_test",
+    data = [":configapi"],
+    substitutions = {
+        "@@CONFIGAPI_CLI@@": "$(rootpath :configapi)",
+    },
+    template = "e2e/test.sh.in",
+)

hack/configapi/delete.go

@@ -7,6 +7,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
@@ -44,7 +45,7 @@ func (d deleteCmd) delete(cmd *cobra.Command) error {
 	return d.attestationClient.DeleteAzureSEVSNPVersion(cmd.Context(), version)
 }
 
-func runDelete(cmd *cobra.Command, _ []string) error {
+func runDelete(cmd *cobra.Command, _ []string) (retErr error) {
 	log := logger.New(logger.PlainLog, zap.DebugLevel).Named("attestationconfigapi")
 
 	region, err := cmd.Flags().GetString("region")
@@ -57,19 +58,27 @@ func runDelete(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("getting bucket: %w", err)
 	}
 
+	distribution, err := cmd.Flags().GetString("distribution")
+	if err != nil {
+		return fmt.Errorf("getting distribution: %w", err)
+	}
+
 	cfg := staticupload.Config{
 		Bucket:         bucket,
 		Region:         region,
+		DistributionID: distribution,
 	}
-	client, stop, err := attestationconfigapi.NewClient(cmd.Context(), cfg, []byte(cosignPwd), []byte(privateKey), false, log)
+	client, clientClose, err := attestationconfigapi.NewClient(cmd.Context(), cfg, []byte(cosignPwd), []byte(privateKey), false, log)
 	if err != nil {
 		return fmt.Errorf("create attestation client: %w", err)
 	}
 	defer func() {
-		if err := stop(cmd.Context()); err != nil {
+		err := clientClose(cmd.Context())
-			cmd.Printf("stopping client: %s\n", err.Error())
+		if err != nil {
+			retErr = errors.Join(retErr, fmt.Errorf("failed to invalidate cache: %w", err))
 		}
 	}()
+
 	deleteCmd := deleteCmd{
 		attestationClient: client,
 	}

hack/configapi/e2e/test.sh.in

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+# Try to upload a file to S3 and then delete it using the configapi cli.
+# Check the file exists after uploading it.
+# Check the file does not exist after deleting it.
+
+###### script header ######
+
+lib=$(realpath @@BASE_LIB@@) || exit 1
+stat "${lib}" >> /dev/null || exit 1
+
+# shellcheck source=../../../bazel/sh/lib.bash
+if ! source "${lib}"; then
+  echo "Error: could not find import"
+  exit 1
+fi
+
+configapi_cli=$(realpath @@CONFIGAPI_CLI@@)
+stat "${configapi_cli}" >> /dev/null
+
+###### script body ######
+
+readonly region="eu-west-1"
+readonly bucket="resource-api-testing"
+readonly distribution="ETZGUP1CWRC2P"
+
+tmpdir=$(mktemp -d)
+readonly tmpdir
+registerExitHandler "rm -rf $tmpdir"
+
+readonly claim_path="'$tmpdir'/maaClaim.json"
+cat << EOF > "$claim_path"
+{
+  "x-ms-isolation-tee": {
+    "x-ms-sevsnpvm-tee-svn": 1,
+    "x-ms-sevsnpvm-snpfw-svn": 9,
+    "x-ms-sevsnpvm-microcode-svn": 116,
+    "x-ms-sevsnpvm-bootloader-svn": 4
+  }
+}
+EOF
+
+readonly date="2023-02-02-03-04"
+${configapi_cli} --maa-claims-path "$claim_path" --upload-date "$date" --region "$region" --bucket "$bucket" --distribution "$distribution"
+
+baseurl="https://d33dzgxuwsgbpw.cloudfront.net/constellation/v1/attestation/azure-sev-snp"
+if ! curl -fsSL ${baseurl}/${date}.json > /dev/null; then
+  echo "Checking for uploaded version file constellation/v1/attestation/azure-sev-snp/${date}.json: request returned ${?}"
+  exit 1
+fi
+
+if ! curl -fsSL ${baseurl}/${date}.json.sig > /dev/null; then
+  echo "Checking for uploaded version signature file constellation/v1/attestation/azure-sev-snp/${date}.json.sig: request returned ${?}"
+  exit 1
+fi
+
+if ! curl -fsSL ${baseurl}/list > /dev/null; then
+  echo "Checking for uploaded list file constellation/v1/attestation/azure-sev-snp/list: request returned ${?}"
+  exit 1
+fi
+${configapi_cli} delete --version "$date" --region "$region" --bucket "$bucket" --distribution "$distribution"
+
+# Omit -f to check for 404. We want to check that a file was deleted, therefore we expect the query to fail.
+http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null ${baseurl}/${date}.json)
+if [[ $http_code -ne 404 ]]; then
+  echo "Expected HTTP code 404 for: constellation/v1/attestation/azure-sev-snp/${date}.json, but got ${http_code}"
+  exit 1
+fi
+# Omit -f to check for 404. We want to check that a file was deleted, therefore we expect the query to fail.
+http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null ${baseurl}/${date}.json.sig)
+if [[ $http_code -ne 404 ]]; then
+  echo "Expected HTTP code 404 for: constellation/v1/attestation/azure-sev-snp/${date}.json, but got ${http_code}"
+  exit 1
+fi

hack/configapi/main.go

@@ -4,6 +4,12 @@ Copyright (c) Edgeless Systems GmbH
 SPDX-License-Identifier: AGPL-3.0-only
 */
 
+/*
+# configapi CLI
+
+A CLI to interact with the Attestationconfig API, a sub API of the Resource API.
+You can execute an e2e test by running: `bazel run //hack/configapi:configapi_e2e_test`.
+*/
 package main
 
 import (
@@ -13,6 +19,7 @@ import (
 	"time"
 
 	"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
+	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/edgelesssys/constellation/v2/internal/logger"
 	"go.uber.org/zap"
 
@@ -23,6 +30,7 @@ import (
 const (
 	awsRegion           = "eu-central-1"
 	awsBucket           = "cdn-constellation-backend"
+	distributionID      = constants.CDNDefaultDistributionID
 	envCosignPwd        = "COSIGN_PASSWORD"
 	envCosignPrivateKey = "COSIGN_PRIVATE_KEY"
 )
@@ -58,6 +66,7 @@ func newRootCmd() *cobra.Command {
 	rootCmd.Flags().StringP("upload-date", "d", "", "upload a version with this date as version name.")
 	rootCmd.PersistentFlags().StringP("region", "r", awsRegion, "region of the targeted bucket.")
 	rootCmd.PersistentFlags().StringP("bucket", "b", awsBucket, "bucket targeted by all operations.")
+	rootCmd.PersistentFlags().StringP("distribution", "i", distributionID, "cloudflare distribution used.")
 	must(rootCmd.MarkFlagRequired("maa-claims-path"))
 	rootCmd.AddCommand(newDeleteCmd())
 	return rootCmd
@@ -72,7 +81,7 @@ func envCheck(_ *cobra.Command, _ []string) error {
 	return nil
 }
 
-func runCmd(cmd *cobra.Command, _ []string) error {
+func runCmd(cmd *cobra.Command, _ []string) (retErr error) {
 	ctx := cmd.Context()
 	log := logger.New(logger.PlainLog, zap.DebugLevel).Named("attestationconfigapi")
 
@@ -84,6 +93,7 @@ func runCmd(cmd *cobra.Command, _ []string) error {
 	cfg := staticupload.Config{
 		Bucket:         flags.bucket,
 		Region:         flags.region,
+		DistributionID: flags.distribution,
 	}
 
 	log.Infof("Reading MAA claims from file: %s", flags.maaFilePath)
@@ -114,17 +124,19 @@ func runCmd(cmd *cobra.Command, _ []string) error {
 	}
 	log.Infof("Input version: %+v is newer than latest API version: %+v", inputVersion, latestAPIVersion)
 
-	client, stop, err := attestationconfigapi.NewClient(ctx, cfg, []byte(cosignPwd), []byte(privateKey), false, log)
+	client, clientClose, err := attestationconfigapi.NewClient(ctx, cfg, []byte(cosignPwd), []byte(privateKey), false, log)
-	defer func() {
+	defer func(retErr *error) {
-		if err := stop(ctx); err != nil {
+		log.Infof("Invalidating cache. This may take some time")
-			cmd.Printf("stopping client: %v\n", err)
+		if err := clientClose(cmd.Context()); err != nil && retErr == nil {
+			*retErr = fmt.Errorf("invalidating cache: %w", err)
 		}
-	}()
+	}(&retErr)
+
 	if err != nil {
 		return fmt.Errorf("creating client: %w", err)
 	}
 
-	if err := client.UploadAzureSEVSNP(ctx, inputVersion, flags.uploadDate); err != nil {
+	if err := client.UploadAzureSEVSNPVersion(ctx, inputVersion, flags.uploadDate); err != nil {
 		return fmt.Errorf("uploading version: %w", err)
 	}
 
@@ -137,6 +149,7 @@ type cliFlags struct {
 	uploadDate   time.Time
 	region       string
 	bucket       string
+	distribution string
 }
 
 func parseCliFlags(cmd *cobra.Command) (cliFlags, error) {
@@ -167,11 +180,17 @@ func parseCliFlags(cmd *cobra.Command) (cliFlags, error) {
 		return cliFlags{}, fmt.Errorf("getting bucket: %w", err)
 	}
 
+	distribution, err := cmd.Flags().GetString("distribution")
+	if err != nil {
+		return cliFlags{}, fmt.Errorf("getting distribution: %w", err)
+	}
+
 	return cliFlags{
 		maaFilePath:  maaFilePath,
 		uploadDate:   uploadDate,
 		region:       region,
 		bucket:       bucket,
+		distribution: distribution,
 	}, nil
 }
 

internal/api/attestationconfigapi/test/BUILD.bazel

@@ -1,13 +0,0 @@
-load("//bazel/go:go_test.bzl", "go_test")
-
-go_test(
-    name = "test_test",
-    srcs = ["integration_test.go"],
-    deps = [
-        "//internal/api/attestationconfigapi",
-        "//internal/logger",
-        "//internal/staticupload",
-        "@com_github_stretchr_testify//require",
-        "@org_uber_go_zap//:zap",
-    ],
-)

internal/api/attestationconfigapi/test/integration_test.go

@@ -1,83 +0,0 @@
-//go:build integration
-
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-package test
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"io"
-	"os"
-	"testing"
-	"time"
-
-	attestationconfig "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
-	"github.com/edgelesssys/constellation/v2/internal/logger"
-	"github.com/edgelesssys/constellation/v2/internal/staticupload"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/zap"
-)
-
-const (
-	awsBucket   = "cdn-constellation-backend"
-	awsRegion   = "eu-central-1"
-	envAwsKeyID = "AWS_ACCESS_KEY_ID"
-	envAwsKey   = "AWS_ACCESS_KEY"
-)
-
-var cfg staticupload.Config
-
-var (
-	cosignPwd      = flag.String("cosign-pwd", "", "Password to decrypt the cosign private key. Required for signing.")
-	privateKeyPath = flag.String("private-key", "", "Path to the private key used for signing. Required for signing.")
-	privateKey     []byte
-)
-
-func TestMain(m *testing.M) {
-	flag.Parse()
-	if *cosignPwd == "" || *privateKeyPath == "" {
-		flag.Usage()
-		fmt.Println("Required flags not set: --cosign-pwd, --private-key. Skipping tests.")
-		os.Exit(1)
-	}
-	if _, present := os.LookupEnv(envAwsKey); !present {
-		fmt.Printf("%s not set. Skipping tests.\n", envAwsKey)
-		os.Exit(1)
-	}
-	if _, present := os.LookupEnv(envAwsKeyID); !present {
-		fmt.Printf("%s not set. Skipping tests.\n", envAwsKeyID)
-		os.Exit(1)
-	}
-	cfg = staticupload.Config{
-		Bucket: awsBucket,
-		Region: awsRegion,
-	}
-	file, _ := os.Open(*privateKeyPath)
-	var err error
-	privateKey, err = io.ReadAll(file)
-	if err != nil {
-		panic(err)
-	}
-	os.Exit(m.Run())
-}
-
-var versionValues = attestationconfig.AzureSEVSNPVersion{
-	Bootloader: 2,
-	TEE:        0,
-	SNP:        6,
-	Microcode:  93,
-}
-
-func TestUploadAzureSEVSNPVersions(t *testing.T) {
-	ctx := context.Background()
-	client, clientClose, err := attestationconfig.NewClient(ctx, cfg, []byte(*cosignPwd), privateKey, false, logger.New(logger.PlainLog, zap.DebugLevel).Named("attestationconfig"))
-	require.NoError(t, err)
-	defer func() { _ = clientClose(ctx) }()
-	d := time.Date(2021, 1, 1, 1, 1, 1, 1, time.UTC)
-	require.NoError(t, client.UploadAzureSEVSNP(ctx, versionValues, d))
-}

internal/staticupload/staticupload.go

@@ -44,6 +44,7 @@ type Client struct {
 	dirtyKeys []string
 	// invalidationIDs is a list of invalidation IDs that are currently in progress.
 	invalidationIDs []string
+	logger          *logger.Logger
 }
 
 // Config is the configuration for the Client.
@@ -118,6 +119,7 @@ func New(ctx context.Context, config Config) (*Client, CloseFunc, error) {
 		cacheInvalidationStrategy:    config.CacheInvalidationStrategy,
 		cacheInvalidationWaitTimeout: config.CacheInvalidationWaitTimeout,
 		bucketID:                     config.Bucket,
+		logger:                       log,
 	}
 	return client, client.Flush, nil
 }
@@ -218,10 +220,13 @@ func (c *Client) waitForInvalidations(ctx context.Context) error {
 			DistributionId: &c.distributionID,
 			Id:             &invalidationID,
 		}
+		c.logger.Debugf("Waiting for invalidation %s in distribution %s", invalidationID, c.distributionID)
 		if err := waiter.Wait(ctx, waitIn, c.cacheInvalidationWaitTimeout); err != nil {
 			return NewInvalidationError(fmt.Errorf("waiting for invalidation to complete: %w", err))
 		}
+
 	}
+	c.logger.Debugf("Invalidations finished")
 	c.invalidationIDs = nil
 	return nil
 }