move attestationconfigapi to public

This commit is contained in:
Adrian Stobbe 2024-09-05 10:03:20 +02:00
parent 3b416224ca
commit 9f4dd3ad21
38 changed files with 29 additions and 29 deletions

View file

@ -0,0 +1,34 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("//bazel/go:go_test.bzl", "go_test")
go_library(
name = "attestationconfigapi",
srcs = [
"attestationconfigapi.go",
"fetcher.go",
"version.go",
],
importpath = "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi",
visibility = ["//:__subpackages__"],
deps = [
"//internal/api/fetcher",
"//internal/attestation/variant",
"//internal/constants",
"//internal/sigstore",
],
)
go_test(
name = "attestationconfigapi_test",
srcs = [
"fetcher_test.go",
"version_test.go",
],
embed = [":attestationconfigapi"],
deps = [
"//internal/attestation/variant",
"//internal/constants",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
],
)

View file

@ -0,0 +1,23 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
/*
# AttestationConfig API
The AttestationConfig API provides values for the attestation key in the Constellation config.
This package defines API types that represents objects of the AttestationConfig API.
The types provide helper methods for validation and commonly used operations on the
information contained in the objects. Especially the paths used for the API are defined
in these helper methods.
Regarding the decision to implement new types over using the existing types from internal/config:
AttestationCfg objects for AttestationCfg API need to hold some version information (for sorting, recognizing latest).
Thus, existing config types (AWSNitroTPM, AzureSEVSNP, ...) can not be extended to implement apiObject interface.
Instead, we need a separate type that wraps _all_ attestation types. In the codebase this is done using the AttestationCfg interface.
The new type AttestationCfgGet needs to be located inside internal/config in order to implement UnmarshalJSON.
*/
package attestationconfigapi

View file

@ -0,0 +1,47 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
load("//bazel/sh:def.bzl", "sh_template")
go_binary(
name = "cli",
embed = [":cli_lib"],
visibility = ["//visibility:public"],
)
go_library(
name = "cli_lib",
srcs = [
"compare.go",
"delete.go",
"main.go",
"upload.go",
"validargs.go",
],
importpath = "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi/cli",
visibility = ["//visibility:private"],
deps = [
"//internal/api/attestationconfigapi",
"//internal/api/attestationconfigapi/cli/client",
"//internal/api/fetcher",
"//internal/attestation/variant",
"//internal/constants",
"//internal/file",
"//internal/logger",
"//internal/staticupload",
"//internal/verify",
"@com_github_aws_aws_sdk_go_v2//aws",
"@com_github_aws_aws_sdk_go_v2_service_s3//:s3",
"@com_github_aws_aws_sdk_go_v2_service_s3//types",
"@com_github_google_go_tdx_guest//proto/tdx",
"@com_github_spf13_afero//:afero",
"@com_github_spf13_cobra//:cobra",
],
)
sh_template(
name = "cli_e2e_test",
data = [":cli"],
substitutions = {
"@@CONFIGAPI_CLI@@": "$(rootpath :cli)",
},
template = "e2e/test.sh.in",
)

View file

@ -0,0 +1,34 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("//bazel/go:go_test.bzl", "go_test")
go_library(
name = "client",
srcs = [
"client.go",
"reporter.go",
],
importpath = "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi/cli/client",
visibility = ["//:__subpackages__"],
deps = [
"//internal/api/attestationconfigapi",
"//internal/api/client",
"//internal/attestation/variant",
"//internal/sigstore",
"//internal/staticupload",
"@com_github_aws_aws_sdk_go//aws",
"@com_github_aws_aws_sdk_go_v2_service_s3//:s3",
],
)
go_test(
name = "client_test",
srcs = [
"client_test.go",
"reporter_test.go",
],
embed = [":client"],
deps = [
"//internal/api/attestationconfigapi",
"@com_github_stretchr_testify//assert",
],
)

View file

@ -0,0 +1,178 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
/*
package client contains code to manage CVM versions in Constellation's CDN API.
It is used to upload and delete "latest" versions for AMD SEV-SNP and Intel TDX.
*/
package client
import (
"context"
"errors"
"fmt"
"log/slog"
"path"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go/aws"
"github.com/edgelesssys/constellation/v2/api/attestationconfigapi"
apiclient "github.com/edgelesssys/constellation/v2/internal/api/client"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/sigstore"
"github.com/edgelesssys/constellation/v2/internal/staticupload"
)
// VersionFormat is the format of the version name in the S3 bucket.
const VersionFormat = "2006-01-02-15-04"
// Client manages (modifies) the version information for the attestation variants.
type Client struct {
s3Client *apiclient.Client
s3ClientClose func(ctx context.Context) error
bucketID string
signer sigstore.Signer
cacheWindowSize int
log *slog.Logger
}
// New returns a new Client.
func New(ctx context.Context, cfg staticupload.Config, cosignPwd, privateKey []byte, dryRun bool, versionWindowSize int, log *slog.Logger) (*Client, apiclient.CloseFunc, error) {
s3Client, clientClose, err := apiclient.NewClient(ctx, cfg.Region, cfg.Bucket, cfg.DistributionID, dryRun, log)
if err != nil {
return nil, nil, fmt.Errorf("failed to create s3 storage: %w", err)
}
repo := &Client{
s3Client: s3Client,
s3ClientClose: clientClose,
signer: sigstore.NewSigner(cosignPwd, privateKey),
bucketID: cfg.Bucket,
cacheWindowSize: versionWindowSize,
log: log,
}
return repo, clientClose, nil
}
// DeleteVersion deletes the given version (without .json suffix) from the API.
func (c Client) DeleteVersion(ctx context.Context, attestation variant.Variant, versionStr string) error {
versions, err := c.List(ctx, attestation)
if err != nil {
return fmt.Errorf("fetch version list: %w", err)
}
ops, err := c.deleteVersion(versions, versionStr)
if err != nil {
return err
}
return executeAllCmds(ctx, c.s3Client, ops)
}
// List returns the list of versions for the given attestation variant.
func (c Client) List(ctx context.Context, attestation variant.Variant) (attestationconfigapi.List, error) {
versions, err := apiclient.Fetch(ctx, c.s3Client, attestationconfigapi.List{Variant: attestation})
if err != nil {
var notFoundErr *apiclient.NotFoundError
if errors.As(err, &notFoundErr) {
return attestationconfigapi.List{Variant: attestation}, nil
}
return attestationconfigapi.List{}, err
}
versions.Variant = attestation
return versions, nil
}
func (c Client) deleteVersion(versions attestationconfigapi.List, versionStr string) (ops []crudCmd, err error) {
versionStr = versionStr + ".json"
ops = append(ops, deleteCmd{
apiObject: attestationconfigapi.Entry{
Variant: versions.Variant,
Version: versionStr,
},
})
removedVersions, err := removeVersion(versions, versionStr)
if err != nil {
return nil, err
}
ops = append(ops, putCmd{
apiObject: removedVersions,
signer: c.signer,
})
return ops, nil
}
func (c Client) listCachedVersions(ctx context.Context, attestation variant.Variant) ([]string, error) {
list, err := c.s3Client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(c.bucketID),
Prefix: aws.String(reportVersionDir(attestation)),
})
if err != nil {
return nil, fmt.Errorf("list objects: %w", err)
}
var dates []string
for _, obj := range list.Contents {
fileName := path.Base(*obj.Key)
// The cache contains signature and json files
// We only want the json files
if date, ok := strings.CutSuffix(fileName, ".json"); ok {
dates = append(dates, date)
}
}
return dates, nil
}
func removeVersion(list attestationconfigapi.List, versionStr string) (removedVersions attestationconfigapi.List, err error) {
versions := list.List
for i, v := range versions {
if v == versionStr {
if i == len(versions)-1 {
removedVersions = attestationconfigapi.List{List: versions[:i], Variant: list.Variant}
} else {
removedVersions = attestationconfigapi.List{List: append(versions[:i], versions[i+1:]...), Variant: list.Variant}
}
return removedVersions, nil
}
}
return attestationconfigapi.List{}, fmt.Errorf("version %s not found in list %v", versionStr, versions)
}
type crudCmd interface {
Execute(ctx context.Context, c *apiclient.Client) error
}
type deleteCmd struct {
apiObject apiclient.APIObject
}
func (d deleteCmd) Execute(ctx context.Context, c *apiclient.Client) error {
return apiclient.DeleteWithSignature(ctx, c, d.apiObject)
}
type putCmd struct {
apiObject apiclient.APIObject
signer sigstore.Signer
}
func (p putCmd) Execute(ctx context.Context, c *apiclient.Client) error {
return apiclient.SignAndUpdate(ctx, c, p.apiObject, p.signer)
}
func executeAllCmds(ctx context.Context, client *apiclient.Client, cmds []crudCmd) error {
for _, cmd := range cmds {
if err := cmd.Execute(ctx, client); err != nil {
return fmt.Errorf("execute operation %+v: %w", cmd, err)
}
}
return nil
}

View file

@ -0,0 +1,34 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"testing"
"github.com/edgelesssys/constellation/v2/api/attestationconfigapi"
"github.com/stretchr/testify/assert"
)
func TestDeleteAzureSEVSNPVersions(t *testing.T) {
sut := Client{
bucketID: "bucket",
}
versions := attestationconfigapi.List{List: []string{"2023-01-01.json", "2021-01-01.json", "2019-01-01.json"}}
ops, err := sut.deleteVersion(versions, "2021-01-01")
assert := assert.New(t)
assert.NoError(err)
assert.Contains(ops, deleteCmd{
apiObject: attestationconfigapi.Entry{
Version: "2021-01-01.json",
},
})
assert.Contains(ops, putCmd{
apiObject: attestationconfigapi.List{List: []string{"2023-01-01.json", "2019-01-01.json"}},
})
}

View file

@ -0,0 +1,367 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"context"
"encoding/json"
"errors"
"fmt"
"path"
"sort"
"strings"
"time"
"github.com/edgelesssys/constellation/v2/api/attestationconfigapi"
"github.com/edgelesssys/constellation/v2/internal/api/client"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
)
// cachedVersionsSubDir is the subdirectory in the bucket where the cached versions are stored.
const cachedVersionsSubDir = "cached-versions"
// ErrNoNewerVersion is returned if the input version is not newer than the latest API version.
var ErrNoNewerVersion = errors.New("input version is not newer than latest API version")
func reportVersionDir(attestation variant.Variant) string {
return path.Join(attestationconfigapi.AttestationURLPath, attestation.String(), cachedVersionsSubDir)
}
// IsInputNewerThanOtherVersion compares the input version with the other version and returns true if the input version is newer.
// This function panics if the input versions are not TDX or SEV-SNP versions.
func IsInputNewerThanOtherVersion(variant variant.Variant, inputVersion, otherVersion any) bool {
var result bool
actionForVariant(variant,
func() {
input := inputVersion.(attestationconfigapi.TDXVersion)
other := otherVersion.(attestationconfigapi.TDXVersion)
result = isInputNewerThanOtherTDXVersion(input, other)
},
func() {
input := inputVersion.(attestationconfigapi.SEVSNPVersion)
other := otherVersion.(attestationconfigapi.SEVSNPVersion)
result = isInputNewerThanOtherSEVSNPVersion(input, other)
},
)
return result
}
// UploadLatestVersion saves the given version to the cache, determines the smallest
// TCB version in the cache among the last cacheWindowSize versions and updates
// the latest version in the API if there is an update.
// force can be used to bypass the validation logic against the cached versions.
func (c Client) UploadLatestVersion(
ctx context.Context, attestationVariant variant.Variant,
inputVersion, latestVersionInAPI any,
now time.Time, force bool,
) error {
// Validate input versions against configured attestation variant
// This allows us to skip these checks in the individual variant implementations
var err error
actionForVariant(attestationVariant,
func() {
if _, ok := inputVersion.(attestationconfigapi.TDXVersion); !ok {
err = fmt.Errorf("input version %q is not a TDX version", inputVersion)
}
if _, ok := latestVersionInAPI.(attestationconfigapi.TDXVersion); !ok {
err = fmt.Errorf("latest API version %q is not a TDX version", latestVersionInAPI)
}
},
func() {
if _, ok := inputVersion.(attestationconfigapi.SEVSNPVersion); !ok {
err = fmt.Errorf("input version %q is not a SNP version", inputVersion)
}
if _, ok := latestVersionInAPI.(attestationconfigapi.SEVSNPVersion); !ok {
err = fmt.Errorf("latest API version %q is not a SNP version", latestVersionInAPI)
}
},
)
if err != nil {
return err
}
if err := c.addVersionToCache(ctx, attestationVariant, inputVersion, now); err != nil {
return fmt.Errorf("adding version to cache: %w", err)
}
// If force is set, immediately update the latest version to the new version in the API.
if force {
return c.uploadAsLatestVersion(ctx, attestationVariant, inputVersion, now)
}
// Otherwise, check the cached versions and update the latest version in the API if necessary.
versionDates, err := c.listCachedVersions(ctx, attestationVariant)
if err != nil {
return fmt.Errorf("listing existing cached versions: %w", err)
}
if len(versionDates) < c.cacheWindowSize {
c.log.Warn(fmt.Sprintf("Skipping version update, found %d, expected %d reported versions.", len(versionDates), c.cacheWindowSize))
return nil
}
minVersion, minDate, err := c.findMinVersion(ctx, attestationVariant, versionDates)
if err != nil {
return fmt.Errorf("determining minimal version in cache: %w", err)
}
c.log.Info(fmt.Sprintf("Found minimal version: %+v with date: %s", minVersion, minDate))
if !IsInputNewerThanOtherVersion(attestationVariant, minVersion, latestVersionInAPI) {
c.log.Info(fmt.Sprintf("Input version: %+v is not newer than latest API version: %+v. Skipping list update", minVersion, latestVersionInAPI))
return ErrNoNewerVersion
}
c.log.Info(fmt.Sprintf("Input version: %+v is newer than latest API version: %+v", minVersion, latestVersionInAPI))
t, err := time.Parse(VersionFormat, minDate)
if err != nil {
return fmt.Errorf("parsing date: %w", err)
}
if err := c.uploadAsLatestVersion(ctx, attestationVariant, minVersion, t); err != nil {
return fmt.Errorf("uploading as latest version: %w", err)
}
c.log.Info(fmt.Sprintf("Successfully uploaded new %s version: %+v", attestationVariant, minVersion))
return nil
}
// uploadAsLatestVersion uploads the given version and updates the list to set it as the "latest" version.
// The version's name is the UTC timestamp of the date.
// The /list entry stores the version name + .json suffix.
func (c Client) uploadAsLatestVersion(ctx context.Context, variant variant.Variant, inputVersion any, date time.Time) error {
versions, err := c.List(ctx, variant)
if err != nil {
return fmt.Errorf("fetch version list: %w", err)
}
if !variant.Equal(versions.Variant) {
return nil
}
dateStr := date.Format(VersionFormat) + ".json"
var ops []crudCmd
obj := apiVersionObject{version: dateStr, variant: variant, cached: false}
obj.setVersion(inputVersion)
ops = append(ops, putCmd{
apiObject: obj,
signer: c.signer,
})
versions.AddVersion(dateStr)
ops = append(ops, putCmd{
apiObject: versions,
signer: c.signer,
})
return executeAllCmds(ctx, c.s3Client, ops)
}
// addVersionToCache adds the given version to the cache.
func (c Client) addVersionToCache(ctx context.Context, variant variant.Variant, inputVersion any, date time.Time) error {
dateStr := date.Format(VersionFormat) + ".json"
obj := apiVersionObject{version: dateStr, variant: variant, cached: true}
obj.setVersion(inputVersion)
cmd := putCmd{
apiObject: obj,
signer: c.signer,
}
return cmd.Execute(ctx, c.s3Client)
}
// findMinVersion returns the minimal version in the cache among the last cacheWindowSize versions.
func (c Client) findMinVersion(
ctx context.Context, attestationVariant variant.Variant, versionDates []string,
) (any, string, error) {
var getMinimalVersion func() (any, string, error)
actionForVariant(attestationVariant,
func() {
getMinimalVersion = func() (any, string, error) {
return findMinimalVersion[attestationconfigapi.TDXVersion](ctx, attestationVariant, versionDates, c.s3Client, c.cacheWindowSize)
}
},
func() {
getMinimalVersion = func() (any, string, error) {
return findMinimalVersion[attestationconfigapi.SEVSNPVersion](ctx, attestationVariant, versionDates, c.s3Client, c.cacheWindowSize)
}
},
)
return getMinimalVersion()
}
func findMinimalVersion[T attestationconfigapi.TDXVersion | attestationconfigapi.SEVSNPVersion](
ctx context.Context, variant variant.Variant, versionDates []string,
s3Client *client.Client, cacheWindowSize int,
) (T, string, error) {
var minimalVersion *T
var minimalDate string
sort.Sort(sort.Reverse(sort.StringSlice(versionDates))) // sort in reverse order to slice the latest versions
versionDates = versionDates[:cacheWindowSize]
sort.Strings(versionDates) // sort with oldest first to to take the minimal version with the oldest date
for _, date := range versionDates {
obj, err := client.Fetch(ctx, s3Client, apiVersionObject{version: date + ".json", variant: variant, cached: true})
if err != nil {
return *new(T), "", fmt.Errorf("get object: %w", err)
}
obj.variant = variant // variant is not set by Fetch, set it manually
if minimalVersion == nil {
v := obj.getVersion().(T)
minimalVersion = &v
minimalDate = date
continue
}
// If the current minimal version has newer versions than the one we just fetched,
// update the minimal version to the older version.
if IsInputNewerThanOtherVersion(variant, *minimalVersion, obj.getVersion()) {
v := obj.getVersion().(T)
minimalVersion = &v
minimalDate = date
}
}
return *minimalVersion, minimalDate, nil
}
type apiVersionObject struct {
version string `json:"-"`
variant variant.Variant `json:"-"`
cached bool `json:"-"`
snp attestationconfigapi.SEVSNPVersion
tdx attestationconfigapi.TDXVersion
}
func (a apiVersionObject) MarshalJSON() ([]byte, error) {
var res []byte
var err error
actionForVariant(a.variant,
func() {
res, err = json.Marshal(a.tdx)
},
func() {
res, err = json.Marshal(a.snp)
},
)
return res, err
}
func (a *apiVersionObject) UnmarshalJSON(data []byte) error {
errTDX := json.Unmarshal(data, &a.tdx)
errSNP := json.Unmarshal(data, &a.snp)
if errTDX == nil || errSNP == nil {
return nil
}
return fmt.Errorf("trying to unmarshal data into both TDX and SNP versions: %w", errors.Join(errTDX, errSNP))
}
// JSONPath returns the path to the JSON file for the request to the config api.
// This is the path to the cached version in the S3 bucket.
func (a apiVersionObject) JSONPath() string {
if a.cached {
return path.Join(reportVersionDir(a.variant), a.version)
}
return path.Join(attestationconfigapi.AttestationURLPath, a.variant.String(), a.version)
}
// ValidateRequest validates the request.
func (a apiVersionObject) ValidateRequest() error {
if !strings.HasSuffix(a.version, ".json") {
return fmt.Errorf("version has no .json suffix")
}
return nil
}
// Validate is a No-Op.
func (a apiVersionObject) Validate() error {
return nil
}
// getVersion returns the version.
func (a apiVersionObject) getVersion() any {
var res any
actionForVariant(a.variant,
func() {
res = a.tdx
},
func() {
res = a.snp
},
)
return res
}
// setVersion sets the version.
func (a *apiVersionObject) setVersion(version any) {
actionForVariant(a.variant,
func() {
a.tdx = version.(attestationconfigapi.TDXVersion)
},
func() {
a.snp = version.(attestationconfigapi.SEVSNPVersion)
},
)
}
// actionForVariant performs the given action based on the whether variant is a TDX or SEV-SNP variant.
func actionForVariant(
attestationVariant variant.Variant,
tdxAction func(), snpAction func(),
) {
switch attestationVariant {
case variant.AWSSEVSNP{}, variant.AzureSEVSNP{}, variant.GCPSEVSNP{}:
snpAction()
case variant.AzureTDX{}:
tdxAction()
default:
panic(fmt.Sprintf("unsupported attestation variant: %s", attestationVariant))
}
}
// isInputNewerThanOtherSEVSNPVersion compares all version fields and returns false if any input field is older, or the versions are equal.
func isInputNewerThanOtherSEVSNPVersion(input, other attestationconfigapi.SEVSNPVersion) bool {
if input == other {
return false
}
if input.TEE < other.TEE {
return false
}
if input.SNP < other.SNP {
return false
}
if input.Microcode < other.Microcode {
return false
}
if input.Bootloader < other.Bootloader {
return false
}
return true
}
// isInputNewerThanOtherSEVSNPVersion compares all version fields and returns false if any input field is older, or the versions are equal.
func isInputNewerThanOtherTDXVersion(input, other attestationconfigapi.TDXVersion) bool {
if input == other {
return false
}
if input.PCESVN < other.PCESVN {
return false
}
if input.QESVN < other.QESVN {
return false
}
// Validate component-wise security version numbers
for idx, inputVersion := range input.TEETCBSVN {
if inputVersion < other.TEETCBSVN[idx] {
return false
}
}
return true
}

View file

@ -0,0 +1,138 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
import (
"testing"
"github.com/edgelesssys/constellation/v2/api/attestationconfigapi"
"github.com/stretchr/testify/assert"
)
func TestIsInputNewerThanOtherSEVSNPVersion(t *testing.T) {
newTestCfg := func() attestationconfigapi.SEVSNPVersion {
return attestationconfigapi.SEVSNPVersion{
Microcode: 93,
TEE: 0,
SNP: 6,
Bootloader: 2,
}
}
testCases := map[string]struct {
latest attestationconfigapi.SEVSNPVersion
input attestationconfigapi.SEVSNPVersion
expect bool
}{
"input is older than latest": {
input: func(c attestationconfigapi.SEVSNPVersion) attestationconfigapi.SEVSNPVersion {
c.Microcode--
return c
}(newTestCfg()),
latest: newTestCfg(),
expect: false,
},
"input has greater and smaller version field than latest": {
input: func(c attestationconfigapi.SEVSNPVersion) attestationconfigapi.SEVSNPVersion {
c.Microcode++
c.Bootloader--
return c
}(newTestCfg()),
latest: newTestCfg(),
expect: false,
},
"input is newer than latest": {
input: func(c attestationconfigapi.SEVSNPVersion) attestationconfigapi.SEVSNPVersion {
c.TEE++
return c
}(newTestCfg()),
latest: newTestCfg(),
expect: true,
},
"input is equal to latest": {
input: newTestCfg(),
latest: newTestCfg(),
expect: false,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
isNewer := isInputNewerThanOtherSEVSNPVersion(tc.input, tc.latest)
assert.Equal(t, tc.expect, isNewer)
})
}
}
func TestIsInputNewerThanOtherTDXVersion(t *testing.T) {
newTestVersion := func() attestationconfigapi.TDXVersion {
return attestationconfigapi.TDXVersion{
QESVN: 1,
PCESVN: 2,
TEETCBSVN: [16]byte{2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2},
QEVendorID: [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
XFAM: [8]byte{0, 1, 2, 3, 4, 5, 6, 7},
}
}
testCases := map[string]struct {
latest attestationconfigapi.TDXVersion
input attestationconfigapi.TDXVersion
expect bool
}{
"input is older than latest": {
input: func(c attestationconfigapi.TDXVersion) attestationconfigapi.TDXVersion {
c.QESVN--
return c
}(newTestVersion()),
latest: newTestVersion(),
expect: false,
},
"input has greater and smaller version field than latest": {
input: func(c attestationconfigapi.TDXVersion) attestationconfigapi.TDXVersion {
c.QESVN++
c.PCESVN--
return c
}(newTestVersion()),
latest: newTestVersion(),
expect: false,
},
"input is newer than latest": {
input: func(c attestationconfigapi.TDXVersion) attestationconfigapi.TDXVersion {
c.QESVN++
return c
}(newTestVersion()),
latest: newTestVersion(),
expect: true,
},
"input is equal to latest": {
input: newTestVersion(),
latest: newTestVersion(),
expect: false,
},
"tee tcb svn is newer": {
input: func(c attestationconfigapi.TDXVersion) attestationconfigapi.TDXVersion {
c.TEETCBSVN[4]++
return c
}(newTestVersion()),
latest: newTestVersion(),
expect: true,
},
"xfam is different": {
input: func(c attestationconfigapi.TDXVersion) attestationconfigapi.TDXVersion {
c.XFAM[3]++
return c
}(newTestVersion()),
latest: newTestVersion(),
expect: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
isNewer := isInputNewerThanOtherTDXVersion(tc.input, tc.latest)
assert.Equal(t, tc.expect, isNewer)
})
}
}

View file

@ -0,0 +1,101 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package main
import (
"fmt"
"os"
"slices"
"github.com/edgelesssys/constellation/v2/api/attestationconfigapi/cli/client"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/verify"
"github.com/google/go-tdx-guest/proto/tdx"
"github.com/spf13/afero"
"github.com/spf13/cobra"
)
func newCompareCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "compare VARIANT FILE [FILE...]",
Short: "Returns the minimum version of all given attestation reports.",
Long: "Compare a list of attestation reports and return the report with the minimum version.",
Example: "cli compare azure-sev-snp report1.json report2.json",
Args: cobra.MatchAll(cobra.MinimumNArgs(2), arg0isAttestationVariant()),
RunE: runCompare,
}
return cmd
}
func runCompare(cmd *cobra.Command, args []string) error {
cmd.SetOut(os.Stdout)
variant, err := variant.FromString(args[0])
if err != nil {
return fmt.Errorf("parsing variant: %w", err)
}
return compare(cmd, variant, args[1:], file.NewHandler(afero.NewOsFs()))
}
func compare(cmd *cobra.Command, attestationVariant variant.Variant, files []string, fs file.Handler) (retErr error) {
if !slices.Contains([]variant.Variant{variant.AWSSEVSNP{}, variant.AzureSEVSNP{}, variant.GCPSEVSNP{}, variant.AzureTDX{}}, attestationVariant) {
return fmt.Errorf("variant %s not supported", attestationVariant)
}
lowestVersion, err := compareVersions(attestationVariant, files, fs)
if err != nil {
return fmt.Errorf("comparing versions: %w", err)
}
cmd.Println(lowestVersion)
return nil
}
func compareVersions(attestationVariant variant.Variant, files []string, fs file.Handler) (string, error) {
readReport := readSNPReport
if attestationVariant.Equal(variant.AzureTDX{}) {
readReport = readTDXReport
}
lowestVersion := files[0]
lowestReport, err := readReport(files[0], fs)
if err != nil {
return "", fmt.Errorf("reading report: %w", err)
}
for _, file := range files[1:] {
report, err := readReport(file, fs)
if err != nil {
return "", fmt.Errorf("reading report: %w", err)
}
if client.IsInputNewerThanOtherVersion(attestationVariant, lowestReport, report) {
lowestVersion = file
lowestReport = report
}
}
return lowestVersion, nil
}
func readSNPReport(file string, fs file.Handler) (any, error) {
var report verify.Report
if err := fs.ReadJSON(file, &report); err != nil {
return nil, fmt.Errorf("reading snp report: %w", err)
}
return convertTCBVersionToSNPVersion(report.SNPReport.LaunchTCB), nil
}
func readTDXReport(file string, fs file.Handler) (any, error) {
var report *tdx.QuoteV4
if err := fs.ReadJSON(file, &report); err != nil {
return nil, fmt.Errorf("reading tdx report: %w", err)
}
return convertQuoteToTDXVersion(report), nil
}

View file

@ -0,0 +1,196 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package main
import (
"context"
"errors"
"fmt"
"log/slog"
"path"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/edgelesssys/constellation/v2/api/attestationconfigapi"
"github.com/edgelesssys/constellation/v2/api/attestationconfigapi/cli/client"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/staticupload"
"github.com/spf13/cobra"
)
// newDeleteCmd creates the delete command.
func newDeleteCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "delete VARIANT KIND <version>",
Short: "Delete an object from the attestationconfig API",
Long: "Delete a specific object version from the config api. <version> is the name of the object to delete (without .json suffix)",
Example: "COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY=$CKEY cli delete azure-sev-snp attestation-report 1.0.0",
Args: cobra.MatchAll(cobra.ExactArgs(3), arg0isAttestationVariant(), isValidKind(1)),
PreRunE: envCheck,
RunE: runDelete,
}
recursivelyCmd := &cobra.Command{
Use: "recursive {aws-sev-snp|azure-sev-snp|azure-tdx|gcp-sev-snp}",
Short: "delete all objects from the API path constellation/v1/attestation/<csp>",
Long: "Delete all objects from the API path constellation/v1/attestation/<csp>",
Example: "COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY=$CKEY cli delete recursive azure-sev-snp",
Args: cobra.MatchAll(cobra.ExactArgs(1), arg0isAttestationVariant()),
RunE: runRecursiveDelete,
}
cmd.AddCommand(recursivelyCmd)
return cmd
}
func runDelete(cmd *cobra.Command, args []string) (retErr error) {
log := logger.NewTextLogger(slog.LevelDebug).WithGroup("attestationconfigapi")
deleteCfg, err := newDeleteConfig(cmd, ([3]string)(args[:3]))
if err != nil {
return fmt.Errorf("creating delete config: %w", err)
}
cfg := staticupload.Config{
Bucket: deleteCfg.bucket,
Region: deleteCfg.region,
DistributionID: deleteCfg.distribution,
}
client, clientClose, err := client.New(cmd.Context(), cfg,
[]byte(cosignPwd), []byte(privateKey), false, 1, log)
if err != nil {
return fmt.Errorf("create attestation client: %w", err)
}
defer func() {
err := clientClose(cmd.Context())
if err != nil {
retErr = errors.Join(retErr, fmt.Errorf("failed to invalidate cache: %w", err))
}
}()
return deleteEntry(cmd.Context(), client, deleteCfg)
}
func runRecursiveDelete(cmd *cobra.Command, args []string) (retErr error) {
// newDeleteConfig expects 3 args, so we pass "all" for the version argument and "snp-report" as kind.
args = append(args, "snp-report")
args = append(args, "all")
deleteCfg, err := newDeleteConfig(cmd, ([3]string)(args[:3]))
if err != nil {
return fmt.Errorf("creating delete config: %w", err)
}
log := logger.NewTextLogger(slog.LevelDebug).WithGroup("attestationconfigapi")
client, closeFn, err := staticupload.New(cmd.Context(), staticupload.Config{
Bucket: deleteCfg.bucket,
Region: deleteCfg.region,
DistributionID: deleteCfg.distribution,
}, log)
if err != nil {
return fmt.Errorf("create static upload client: %w", err)
}
defer func() {
err := closeFn(cmd.Context())
if err != nil {
retErr = errors.Join(retErr, fmt.Errorf("failed to close client: %w", err))
}
}()
deletePath := path.Join(attestationconfigapi.AttestationURLPath, deleteCfg.variant.String())
return deleteEntryRecursive(cmd.Context(), deletePath, client, deleteCfg)
}
type deleteConfig struct {
variant variant.Variant
kind objectKind
version string
region string
bucket string
url string
distribution string
cosignPublicKey string
}
func newDeleteConfig(cmd *cobra.Command, args [3]string) (deleteConfig, error) {
region, err := cmd.Flags().GetString("region")
if err != nil {
return deleteConfig{}, fmt.Errorf("getting region: %w", err)
}
bucket, err := cmd.Flags().GetString("bucket")
if err != nil {
return deleteConfig{}, fmt.Errorf("getting bucket: %w", err)
}
testing, err := cmd.Flags().GetBool("testing")
if err != nil {
return deleteConfig{}, fmt.Errorf("getting testing flag: %w", err)
}
apiCfg := getAPIEnvironment(testing)
variant, err := variant.FromString(args[0])
if err != nil {
return deleteConfig{}, fmt.Errorf("invalid attestation variant: %q: %w", args[0], err)
}
kind := kindFromString(args[1])
version := args[2]
return deleteConfig{
variant: variant,
kind: kind,
version: version,
region: region,
bucket: bucket,
url: apiCfg.url,
distribution: apiCfg.distribution,
cosignPublicKey: apiCfg.cosignPublicKey,
}, nil
}
func deleteEntry(ctx context.Context, client *client.Client, cfg deleteConfig) error {
if cfg.kind != attestationReport {
return fmt.Errorf("kind %s not supported", cfg.kind)
}
return client.DeleteVersion(ctx, cfg.variant, cfg.version)
}
func deleteEntryRecursive(ctx context.Context, path string, client *staticupload.Client, cfg deleteConfig) error {
resp, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(cfg.bucket),
Prefix: aws.String(path),
})
if err != nil {
return err
}
// Delete all objects in the path.
objIDs := make([]s3types.ObjectIdentifier, len(resp.Contents))
for i, obj := range resp.Contents {
objIDs[i] = s3types.ObjectIdentifier{Key: obj.Key}
}
if len(objIDs) > 0 {
_, err = client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
Bucket: aws.String(cfg.bucket),
Delete: &s3types.Delete{
Objects: objIDs,
Quiet: toPtr(true),
},
})
if err != nil {
return err
}
}
return nil
}
func toPtr[T any](v T) *T {
return &v
}

View file

@ -0,0 +1,244 @@
#!/usr/bin/env bash
# Try to upload a file to S3 and then delete it using the configapi cli.
# Check the file exists after uploading it.
# Check the file does not exist after deleting it.
###### script header ######
lib=$(realpath @@BASE_LIB@@) || exit 1
stat "${lib}" >> /dev/null || exit 1
# shellcheck source=../../../../../bazel/sh/lib.bash
if ! source "${lib}"; then
echo "Error: could not find import"
exit 1
fi
configapi_cli=$(realpath @@CONFIGAPI_CLI@@)
stat "${configapi_cli}" >> /dev/null
configapi_cli="${configapi_cli} --testing"
###### script body ######
attestationVariant=$1
readonly attestationVariant
readonly region="eu-west-1"
readonly bucket="resource-api-testing"
tmpdir=$(mktemp -d)
readonly tmpdir
registerExitHandler "rm -rf ${tmpdir}"
# empty the bucket version state
${configapi_cli} delete recursive "${attestationVariant}" --region "${region}" --bucket "${bucket}"
readonly current_report_path="${tmpdir}/attestationReportCurrent.json"
readonly report_path="${tmpdir}/attestationReport.json"
readonly older_report_path="${tmpdir}/attestationReportOld.json"
if [[ ${attestationVariant} == *-tdx ]]; then
cat << EOF > "${current_report_path}"
{
"header": {
"qe_svn": "AAA=",
"pce_svn": "AAA=",
"qe_vendor_id": "KioqKioqKioqKioqKioqKg=="
},
"td_quote_body": {
"tee_tcb_svn": "AAAAAAAAAAAAAAAAAAAAAA==",
"xfam": "AAAAAAAAAAA="
}
}
EOF
# the high version numbers ensure that it's newer than the current latest value
cat << EOF > "${report_path}"
{
"header": {
"qe_svn": "//8=",
"pce_svn": "//8=",
"qe_vendor_id": "KioqKioqKioqKioqKioqKg=="
},
"td_quote_body": {
"tee_tcb_svn": "/////////////////////w==",
"xfam": "AQIDBAUGBwg="
}
}
EOF
# has an older version
cat << EOF > "${older_report_path}"
{
"header": {
"qe_svn": "//8=",
"pce_svn": "/v8=",
"qe_vendor_id": "KioqKioqKioqKioqKioqKg=="
},
"td_quote_body": {
"tee_tcb_svn": "/////////////////////g==",
"xfam": "AQIDBAUGBwg="
}
}
EOF
elif [[ ${attestationVariant} == *-sev-snp ]]; then
cat << EOF > "${current_report_path}"
{
"snp_report": {
"reported_tcb": {
"bootloader": 1,
"tee": 1,
"snp": 1,
"microcode": 1
},
"committed_tcb": {
"bootloader": 1,
"tee": 1,
"snp": 1,
"microcode": 1
},
"launch_tcb": {
"bootloader": 1,
"tee": 1,
"snp": 1,
"microcode": 1
}
}
}
EOF
# the high version numbers ensure that it's newer than the current latest value
cat << EOF > "${report_path}"
{
"snp_report": {
"reported_tcb": {
"bootloader": 255,
"tee": 255,
"snp": 255,
"microcode": 255
},
"committed_tcb": {
"bootloader": 255,
"tee": 255,
"snp": 255,
"microcode": 255
},
"launch_tcb": {
"bootloader": 255,
"tee": 255,
"snp": 255,
"microcode": 255
}
}
}
EOF
# has an older version
cat << EOF > "${older_report_path}"
{
"snp_report": {
"reported_tcb": {
"bootloader": 255,
"tee": 255,
"snp": 255,
"microcode": 254
},
"committed_tcb": {
"bootloader": 255,
"tee": 255,
"snp": 255,
"microcode": 254
},
"launch_tcb": {
"bootloader": 255,
"tee": 255,
"snp": 255,
"microcode": 254
}
}
}
EOF
else
echo "Unknown attestation variant: ${attestationVariant}"
exit 1
fi
# upload a fake latest version for the fetcher
${configapi_cli} upload "${attestationVariant}" attestation-report "${current_report_path}" --force --upload-date "2000-01-01-01-01" --region "${region}" --bucket "${bucket}"
# report 3 versions with different dates to fill the reporter cache
readonly date_oldest="2023-02-01-03-04"
${configapi_cli} upload "${attestationVariant}" attestation-report "${older_report_path}" --upload-date "${date_oldest}" --region "${region}" --bucket "${bucket}" --cache-window-size 3
readonly date_older="2023-02-02-03-04"
${configapi_cli} upload "${attestationVariant}" attestation-report "${older_report_path}" --upload-date "${date_older}" --region "${region}" --bucket "${bucket}" --cache-window-size 3
readonly date="2023-02-03-03-04"
${configapi_cli} upload "${attestationVariant}" attestation-report "${report_path}" --upload-date "${date}" --region "${region}" --bucket "${bucket}" --cache-window-size 3
# expect that $date_oldest is served as latest version
basepath="constellation/v1/attestation/${attestationVariant}"
baseurl="https://d33dzgxuwsgbpw.cloudfront.net/${basepath}"
if ! curl -fsSL "${baseurl}/${date_oldest}.json" > version.json; then
echo "Checking for uploaded version file ${basepath}/${date_oldest}.json: request returned ${?}"
exit 1
fi
if [[ ${attestationVariant} == *-tdx ]]; then
# check that version values are equal to expected
if ! cmp -s <(echo -n '{"qeSVN":65535,"pceSVN":65534,"teeTCBSVN":[255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,254],"qeVendorID":[42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42],"xfam":[1,2,3,4,5,6,7,8]}') version.json; then
echo "The version content:"
cat version.json
echo " is not equal to the expected version content:"
echo '{"qeSVN":65535,"pceSVN":65534,"teeTCBSVN":[255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,254],"qeVendorID":[42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42],"xfam":[1,2,3,4,5,6,7,8]}'
exit 1
fi
elif [[ ${attestationVariant} == *-sev-snp ]]; then
# check that version values are equal to expected
if ! cmp -s <(echo -n '{"bootloader":255,"tee":255,"snp":255,"microcode":254}') version.json; then
echo "The version content:"
cat version.json
echo " is not equal to the expected version content:"
echo '{"bootloader":255,"tee":255,"snp":255,"microcode":254}'
exit 1
fi
fi
if ! curl -fsSL "${baseurl}/${date_oldest}.json.sig" > /dev/null; then
echo "Checking for uploaded version signature file ${basepath}/${date_oldest}.json.sig: request returned ${?}"
exit 1
fi
# check list endpoint
if ! curl -fsSL "${baseurl}"/list > list.json; then
echo "Checking for uploaded list file ${basepath}/list: request returned ${?}"
exit 1
fi
# check that version values are equal to expected
if ! cmp -s <(echo -n '["2023-02-01-03-04.json","2000-01-01-01-01.json"]') list.json; then
echo "The list content:"
cat list.json
echo " is not equal to the expected version content:"
echo '["2023-02-01-03-04.json","2000-01-01-01-01.json"]'
exit 1
fi
# check that the other versions are not uploaded
http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null "${baseurl}/${date_older}.json")
if [[ ${http_code} -ne 404 ]]; then
echo "Expected HTTP code 404 for: ${basepath}/${date_older}.json, but got ${http_code}"
exit 1
fi
http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null "${baseurl}/${date}.json.sig")
if [[ ${http_code} -ne 404 ]]; then
echo "Expected HTTP code 404 for: ${basepath}/${date}.json, but got ${http_code}"
exit 1
fi
${configapi_cli} delete "${attestationVariant}" attestation-report "${date_oldest}" --region "${region}" --bucket "${bucket}"
# Omit -f to check for 404. We want to check that a file was deleted, therefore we expect the query to fail.
http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null "${baseurl}/${date_oldest}.json")
if [[ ${http_code} -ne 404 ]]; then
echo "Expected HTTP code 404 for: ${basepath}/${date_oldest}.json, but got ${http_code}"
exit 1
fi
# Omit -f to check for 404. We want to check that a file was deleted, therefore we expect the query to fail.
http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null "${baseurl}/${date_oldest}.json.sig")
if [[ ${http_code} -ne 404 ]]; then
echo "Expected HTTP code 404 for: ${basepath}/${date_oldest}.json, but got ${http_code}"
exit 1
fi

View file

@ -0,0 +1,78 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
/*
This package provides a CLI to interact with the Attestationconfig API, a sub API of the Resource API.
You can execute an e2e test by running: `bazel run //internal/api/attestationconfigapi:configapi_e2e_test`.
The CLI is used in the CI pipeline. Manual actions that change the bucket's data shouldn't be necessary.
The reporter CLI caches the observed version values in a dedicated caching directory and derives the latest API version from it.
Any version update is then pushed to the API.
*/
package main
import (
"os"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/spf13/cobra"
)
const (
awsRegion = "eu-central-1"
awsBucket = "cdn-constellation-backend"
distributionID = constants.CDNDefaultDistributionID
envCosignPwd = "COSIGN_PASSWORD"
envCosignPrivateKey = "COSIGN_PRIVATE_KEY"
// versionWindowSize defines the number of versions to be considered for the latest version.
// Through our weekly e2e tests, each week 2 versions are uploaded:
// One from a stable release, and one from a debug image.
// A window size of 6 ensures we update only after a version has been "stable" for 3 weeks.
versionWindowSize = 6
)
var (
// Cosign credentials.
cosignPwd string
privateKey string
)
func main() {
if err := newRootCmd().Execute(); err != nil {
os.Exit(1)
}
os.Exit(0)
}
// newRootCmd creates the root command.
func newRootCmd() *cobra.Command {
rootCmd := &cobra.Command{
Short: "CLI to interact with the attestationconfig API",
Long: "CLI to interact with the attestationconfig API. Allows uploading new TCB versions, deleting specific versions and deleting all versions. Uploaded objects are signed with cosign.",
}
rootCmd.PersistentFlags().StringP("region", "r", awsRegion, "region of the targeted bucket.")
rootCmd.PersistentFlags().StringP("bucket", "b", awsBucket, "bucket targeted by all operations.")
rootCmd.PersistentFlags().Bool("testing", false, "upload to S3 test bucket.")
rootCmd.AddCommand(newUploadCmd())
rootCmd.AddCommand(newDeleteCmd())
rootCmd.AddCommand(newCompareCmd())
return rootCmd
}
type apiConfig struct {
url string
distribution string
cosignPublicKey string
}
func getAPIEnvironment(testing bool) apiConfig {
if testing {
return apiConfig{url: "https://d33dzgxuwsgbpw.cloudfront.net", distribution: "ETZGUP1CWRC2P", cosignPublicKey: constants.CosignPublicKeyDev}
}
return apiConfig{url: constants.CDNRepositoryURL, distribution: constants.CDNDefaultDistributionID, cosignPublicKey: constants.CosignPublicKeyReleases}
}

View file

@ -0,0 +1,245 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package main
import (
"context"
"encoding/binary"
"errors"
"fmt"
"log/slog"
"os"
"time"
"github.com/edgelesssys/constellation/v2/api/attestationconfigapi"
"github.com/edgelesssys/constellation/v2/api/attestationconfigapi/cli/client"
"github.com/edgelesssys/constellation/v2/internal/api/fetcher"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/staticupload"
"github.com/edgelesssys/constellation/v2/internal/verify"
"github.com/google/go-tdx-guest/proto/tdx"
"github.com/spf13/afero"
"github.com/spf13/cobra"
)
func newUploadCmd() *cobra.Command {
uploadCmd := &cobra.Command{
Use: "upload VARIANT KIND FILE",
Short: "Upload an object to the attestationconfig API",
Long: fmt.Sprintf("Upload a new object to the attestationconfig API. For snp-reports the new object is added to a cache folder first.\n"+
"The CLI then determines the lowest version within the cache-window present in the cache and writes that value to the config api if necessary. "+
"For guest-firmware objects the object is added to the API directly.\n"+
"Please authenticate with AWS through your preferred method (e.g. environment variables, CLI) "+
"to be able to upload to S3. Set the %s and %s environment variables to authenticate with cosign.",
envCosignPrivateKey, envCosignPwd,
),
Example: "COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY=$CKEY cli upload azure-sev-snp attestation-report /some/path/report.json",
Args: cobra.MatchAll(cobra.ExactArgs(3), arg0isAttestationVariant(), isValidKind(1)),
PreRunE: envCheck,
RunE: runUpload,
}
uploadCmd.Flags().StringP("upload-date", "d", "", "upload a version with this date as version name.")
uploadCmd.Flags().BoolP("force", "f", false, "Use force to manually push a new latest version."+
" The version gets saved to the cache but the version selection logic is skipped.")
uploadCmd.Flags().IntP("cache-window-size", "s", versionWindowSize, "Number of versions to be considered for the latest version.")
return uploadCmd
}
func envCheck(_ *cobra.Command, _ []string) error {
if os.Getenv(envCosignPrivateKey) == "" || os.Getenv(envCosignPwd) == "" {
return fmt.Errorf("please set both %s and %s environment variables", envCosignPrivateKey, envCosignPwd)
}
cosignPwd = os.Getenv(envCosignPwd)
privateKey = os.Getenv(envCosignPrivateKey)
return nil
}
func runUpload(cmd *cobra.Command, args []string) (retErr error) {
ctx := cmd.Context()
log := logger.NewTextLogger(slog.LevelDebug).WithGroup("attestationconfigapi")
uploadCfg, err := newConfig(cmd, ([3]string)(args[:3]))
if err != nil {
return fmt.Errorf("parsing cli flags: %w", err)
}
client, clientClose, err := client.New(ctx,
staticupload.Config{
Bucket: uploadCfg.bucket,
Region: uploadCfg.region,
DistributionID: uploadCfg.distribution,
},
[]byte(cosignPwd), []byte(privateKey),
false, uploadCfg.cacheWindowSize, log,
)
defer func() {
err := clientClose(cmd.Context())
if err != nil {
retErr = errors.Join(retErr, fmt.Errorf("failed to invalidate cache: %w", err))
}
}()
if err != nil {
return fmt.Errorf("creating client: %w", err)
}
return uploadReport(ctx, client, uploadCfg, file.NewHandler(afero.NewOsFs()), log)
}
func uploadReport(
ctx context.Context, apiClient *client.Client,
cfg uploadConfig, fs file.Handler, log *slog.Logger,
) error {
if cfg.kind != attestationReport {
return fmt.Errorf("kind %s not supported", cfg.kind)
}
apiFetcher := attestationconfigapi.NewFetcherWithCustomCDNAndCosignKey(cfg.url, cfg.cosignPublicKey)
latestVersionInAPI, err := apiFetcher.FetchLatestVersion(ctx, cfg.variant)
if err != nil {
var notFoundErr *fetcher.NotFoundError
if errors.As(err, &notFoundErr) {
log.Info("No versions found in API, but assuming that we are uploading the first version.")
} else {
return fmt.Errorf("fetching latest version: %w", err)
}
}
var newVersion, latestVersion any
switch cfg.variant {
case variant.AWSSEVSNP{}, variant.AzureSEVSNP{}, variant.GCPSEVSNP{}:
latestVersion = latestVersionInAPI.SEVSNPVersion
log.Info(fmt.Sprintf("Reading SNP report from file: %s", cfg.path))
newVersion, err = readSNPReport(cfg.path, fs)
if err != nil {
return err
}
log.Info(fmt.Sprintf("Input SNP report: %+v", newVersion))
case variant.AzureTDX{}:
latestVersion = latestVersionInAPI.TDXVersion
log.Info(fmt.Sprintf("Reading TDX report from file: %s", cfg.path))
newVersion, err = readTDXReport(cfg.path, fs)
if err != nil {
return err
}
log.Info(fmt.Sprintf("Input TDX report: %+v", newVersion))
default:
return fmt.Errorf("variant %s not supported", cfg.variant)
}
if err := apiClient.UploadLatestVersion(
ctx, cfg.variant, newVersion, latestVersion, cfg.uploadDate, cfg.force,
); err != nil && !errors.Is(err, client.ErrNoNewerVersion) {
return fmt.Errorf("updating latest version: %w", err)
}
return nil
}
func convertTCBVersionToSNPVersion(tcb verify.TCBVersion) attestationconfigapi.SEVSNPVersion {
return attestationconfigapi.SEVSNPVersion{
Bootloader: tcb.Bootloader,
TEE: tcb.TEE,
SNP: tcb.SNP,
Microcode: tcb.Microcode,
}
}
func convertQuoteToTDXVersion(quote *tdx.QuoteV4) attestationconfigapi.TDXVersion {
return attestationconfigapi.TDXVersion{
QESVN: binary.LittleEndian.Uint16(quote.Header.QeSvn),
PCESVN: binary.LittleEndian.Uint16(quote.Header.PceSvn),
QEVendorID: [16]byte(quote.Header.QeVendorId),
XFAM: [8]byte(quote.TdQuoteBody.Xfam),
TEETCBSVN: [16]byte(quote.TdQuoteBody.TeeTcbSvn),
}
}
type uploadConfig struct {
variant variant.Variant
kind objectKind
path string
uploadDate time.Time
cosignPublicKey string
region string
bucket string
distribution string
url string
force bool
cacheWindowSize int
}
func newConfig(cmd *cobra.Command, args [3]string) (uploadConfig, error) {
dateStr, err := cmd.Flags().GetString("upload-date")
if err != nil {
return uploadConfig{}, fmt.Errorf("getting upload date: %w", err)
}
uploadDate := time.Now()
if dateStr != "" {
uploadDate, err = time.Parse(client.VersionFormat, dateStr)
if err != nil {
return uploadConfig{}, fmt.Errorf("parsing date: %w", err)
}
}
region, err := cmd.Flags().GetString("region")
if err != nil {
return uploadConfig{}, fmt.Errorf("getting region: %w", err)
}
bucket, err := cmd.Flags().GetString("bucket")
if err != nil {
return uploadConfig{}, fmt.Errorf("getting bucket: %w", err)
}
testing, err := cmd.Flags().GetBool("testing")
if err != nil {
return uploadConfig{}, fmt.Errorf("getting testing flag: %w", err)
}
apiCfg := getAPIEnvironment(testing)
force, err := cmd.Flags().GetBool("force")
if err != nil {
return uploadConfig{}, fmt.Errorf("getting force: %w", err)
}
cacheWindowSize, err := cmd.Flags().GetInt("cache-window-size")
if err != nil {
return uploadConfig{}, fmt.Errorf("getting cache window size: %w", err)
}
variant, err := variant.FromString(args[0])
if err != nil {
return uploadConfig{}, fmt.Errorf("invalid attestation variant: %q: %w", args[0], err)
}
kind := kindFromString(args[1])
path := args[2]
return uploadConfig{
variant: variant,
kind: kind,
path: path,
uploadDate: uploadDate,
cosignPublicKey: apiCfg.cosignPublicKey,
region: region,
bucket: bucket,
url: apiCfg.url,
distribution: apiCfg.distribution,
force: force,
cacheWindowSize: cacheWindowSize,
}, nil
}

View file

@ -0,0 +1,60 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package main
import (
"errors"
"fmt"
"strings"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/spf13/cobra"
)
func arg0isAttestationVariant() cobra.PositionalArgs {
return func(_ *cobra.Command, args []string) error {
attestationVariant, err := variant.FromString(args[0])
if err != nil {
return errors.New("argument 0 isn't a valid attestation variant")
}
switch attestationVariant {
case variant.AWSSEVSNP{}, variant.AzureSEVSNP{}, variant.AzureTDX{}, variant.GCPSEVSNP{}:
return nil
default:
return errors.New("argument 0 isn't a supported attestation variant")
}
}
}
func isValidKind(arg int) cobra.PositionalArgs {
return func(_ *cobra.Command, args []string) error {
if kind := kindFromString(args[arg]); kind == unknown {
return fmt.Errorf("argument %s isn't a valid kind: must be one of [%q, %q]", args[arg], attestationReport, guestFirmware)
}
return nil
}
}
// objectKind encodes the available actions.
type objectKind string
const (
// unknown is the default objectKind and does nothing.
unknown objectKind = "unknown-kind"
attestationReport objectKind = "attestation-report"
guestFirmware objectKind = "guest-firmware"
)
func kindFromString(s string) objectKind {
lower := strings.ToLower(s)
switch objectKind(lower) {
case attestationReport, guestFirmware:
return objectKind(lower)
default:
return unknown
}
}

View file

@ -0,0 +1,109 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package attestationconfigapi
import (
"context"
"fmt"
apifetcher "github.com/edgelesssys/constellation/v2/internal/api/fetcher"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/sigstore"
)
const cosignPublicKey = constants.CosignPublicKeyReleases
// Fetcher fetches config API resources without authentication.
type Fetcher interface {
FetchLatestVersion(ctx context.Context, attestation fmt.Stringer) (Entry, error)
}
// fetcher fetches AttestationCfg API resources without authentication.
type fetcher struct {
apifetcher.HTTPClient
cdnURL string
verifier sigstore.Verifier
}
// NewFetcher returns a new apifetcher.
func NewFetcher() Fetcher {
return NewFetcherWithClient(apifetcher.NewHTTPClient(), constants.CDNRepositoryURL)
}
// NewFetcherWithCustomCDNAndCosignKey returns a new fetcher with custom CDN URL.
func NewFetcherWithCustomCDNAndCosignKey(cdnURL, cosignKey string) Fetcher {
verifier, err := sigstore.NewCosignVerifier([]byte(cosignKey))
if err != nil {
// This relies on an embedded public key. If this key can not be validated, there is no way to recover from this.
panic(fmt.Errorf("creating cosign verifier: %w", err))
}
return newFetcherWithClientAndVerifier(apifetcher.NewHTTPClient(), verifier, cdnURL)
}
// NewFetcherWithClient returns a new fetcher with custom http client.
func NewFetcherWithClient(client apifetcher.HTTPClient, cdnURL string) Fetcher {
verifier, err := sigstore.NewCosignVerifier([]byte(cosignPublicKey))
if err != nil {
// This relies on an embedded public key. If this key can not be validated, there is no way to recover from this.
panic(fmt.Errorf("creating cosign verifier: %w", err))
}
return newFetcherWithClientAndVerifier(client, verifier, cdnURL)
}
func newFetcherWithClientAndVerifier(client apifetcher.HTTPClient, cosignVerifier sigstore.Verifier, url string) Fetcher {
return &fetcher{HTTPClient: client, verifier: cosignVerifier, cdnURL: url}
}
// FetchLatestVersion returns the latest versions of the given type.
func (f *fetcher) FetchLatestVersion(ctx context.Context, variant fmt.Stringer) (Entry, error) {
list, err := f.fetchVersionList(ctx, variant)
if err != nil {
return Entry{}, err
}
// latest version is first in list
return f.fetchVersion(ctx, list.List[0], variant)
}
// fetchVersionList fetches the version list information from the config API.
func (f *fetcher) fetchVersionList(ctx context.Context, attestationVariant fmt.Stringer) (List, error) {
parsedVariant, err := variant.FromString(attestationVariant.String())
if err != nil {
return List{}, fmt.Errorf("parsing variant: %w", err)
}
fetchedList, err := apifetcher.FetchAndVerify(ctx, f.HTTPClient, f.cdnURL, List{Variant: parsedVariant}, f.verifier)
if err != nil {
return List{}, fmt.Errorf("fetching version list: %w", err)
}
// Set the attestation variant of the list as it is not part of the marshalled JSON retrieved by Fetch
fetchedList.Variant = parsedVariant
return fetchedList, nil
}
// fetchVersion fetches the version information from the config API.
func (f *fetcher) fetchVersion(ctx context.Context, version string, attestationVariant fmt.Stringer) (Entry, error) {
parsedVariant, err := variant.FromString(attestationVariant.String())
if err != nil {
return Entry{}, fmt.Errorf("parsing variant: %w", err)
}
obj := Entry{
Version: version,
Variant: parsedVariant,
}
fetchedVersion, err := apifetcher.FetchAndVerify(ctx, f.HTTPClient, f.cdnURL, obj, f.verifier)
if err != nil {
return Entry{}, fmt.Errorf("fetching version %q: %w", version, err)
}
// Set the attestation variant of the list as it is not part of the marshalled JSON retrieved by FetchAndVerify
fetchedVersion.Variant = parsedVariant
return fetchedVersion, nil
}

View file

@ -0,0 +1,188 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package attestationconfigapi
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"testing"
"time"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/stretchr/testify/assert"
)
func TestFetchLatestSEVSNPVersion(t *testing.T) {
latestVersionSNP := Entry{
SEVSNPVersion: SEVSNPVersion{
Microcode: 93,
TEE: 0,
SNP: 6,
Bootloader: 2,
},
}
olderVersionSNP := Entry{
SEVSNPVersion: SEVSNPVersion{
Microcode: 1,
TEE: 0,
SNP: 1,
Bootloader: 1,
},
}
latestVersionTDX := Entry{
TDXVersion: TDXVersion{
QESVN: 2,
PCESVN: 3,
TEETCBSVN: [16]byte{4},
QEVendorID: [16]byte{5},
XFAM: [8]byte{6},
},
}
olderVersionTDX := Entry{
TDXVersion: TDXVersion{
QESVN: 1,
PCESVN: 2,
TEETCBSVN: [16]byte{3},
QEVendorID: [16]byte{4},
XFAM: [8]byte{5},
},
}
latestStr := "2023-06-11-14-09.json"
olderStr := "2019-01-01-01-01.json"
testCases := map[string]struct {
fetcherVersions []string
timeAtTest time.Time
wantErr bool
attestation variant.Variant
expectedVersion Entry
olderVersion Entry
latestVersion Entry
}{
"get latest version azure-sev-snp": {
fetcherVersions: []string{latestStr, olderStr},
attestation: variant.AzureSEVSNP{},
expectedVersion: func() Entry { tmp := latestVersionSNP; tmp.Variant = variant.AzureSEVSNP{}; return tmp }(),
olderVersion: func() Entry { tmp := olderVersionSNP; tmp.Variant = variant.AzureSEVSNP{}; return tmp }(),
latestVersion: func() Entry { tmp := latestVersionSNP; tmp.Variant = variant.AzureSEVSNP{}; return tmp }(),
},
"get latest version aws-sev-snp": {
fetcherVersions: []string{latestStr, olderStr},
attestation: variant.AWSSEVSNP{},
expectedVersion: func() Entry { tmp := latestVersionSNP; tmp.Variant = variant.AWSSEVSNP{}; return tmp }(),
olderVersion: func() Entry { tmp := olderVersionSNP; tmp.Variant = variant.AWSSEVSNP{}; return tmp }(),
latestVersion: func() Entry { tmp := latestVersionSNP; tmp.Variant = variant.AWSSEVSNP{}; return tmp }(),
},
"get latest version azure-tdx": {
fetcherVersions: []string{latestStr, olderStr},
attestation: variant.AzureTDX{},
expectedVersion: func() Entry { tmp := latestVersionTDX; tmp.Variant = variant.AzureTDX{}; return tmp }(),
olderVersion: func() Entry { tmp := olderVersionTDX; tmp.Variant = variant.AzureTDX{}; return tmp }(),
latestVersion: func() Entry { tmp := latestVersionTDX; tmp.Variant = variant.AzureTDX{}; return tmp }(),
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
client := &http.Client{
Transport: &fakeConfigAPIHandler{
attestation: tc.attestation,
versions: tc.fetcherVersions,
latestDate: latestStr,
latestVersion: tc.latestVersion,
olderDate: olderStr,
olderVersion: tc.olderVersion,
},
}
fetcher := newFetcherWithClientAndVerifier(client, stubVerifier{}, constants.CDNRepositoryURL)
res, err := fetcher.FetchLatestVersion(context.Background(), tc.attestation)
assert := assert.New(t)
if tc.wantErr {
assert.Error(err)
} else {
assert.NoError(err)
assert.Equal(tc.expectedVersion, res)
}
})
}
}
type fakeConfigAPIHandler struct {
attestation variant.Variant
versions []string
latestDate string
latestVersion Entry
olderDate string
olderVersion Entry
}
// RoundTrip resolves the request and returns a dummy response.
func (f *fakeConfigAPIHandler) RoundTrip(req *http.Request) (*http.Response, error) {
switch req.URL.Path {
case fmt.Sprintf("/constellation/v1/attestation/%s/list", f.attestation.String()):
res := &http.Response{}
bt, err := json.Marshal(f.versions)
if err != nil {
return nil, err
}
res.Body = io.NopCloser(bytes.NewReader(bt))
res.Header = http.Header{}
res.Header.Set("Content-Type", "application/json")
res.StatusCode = http.StatusOK
return res, nil
case fmt.Sprintf("/constellation/v1/attestation/%s/list.sig", f.attestation.String()):
res := &http.Response{}
res.Body = io.NopCloser(bytes.NewReader([]byte("null")))
res.StatusCode = http.StatusOK
return res, nil
case fmt.Sprintf("/constellation/v1/attestation/%s/%s", f.attestation.String(), f.latestDate):
res := &http.Response{}
bt, err := json.Marshal(f.latestVersion)
if err != nil {
return nil, err
}
res.Body = io.NopCloser(bytes.NewReader(bt))
res.StatusCode = http.StatusOK
return res, nil
case fmt.Sprintf("/constellation/v1/attestation/%s/%s", f.attestation.String(), f.olderDate):
res := &http.Response{}
bt, err := json.Marshal(f.olderVersion)
if err != nil {
return nil, err
}
res.Body = io.NopCloser(bytes.NewReader(bt))
res.StatusCode = http.StatusOK
return res, nil
case fmt.Sprintf("/constellation/v1/attestation/%s/%s.sig", f.attestation.String(), f.latestDate):
res := &http.Response{}
res.Body = io.NopCloser(bytes.NewReader([]byte("null")))
res.StatusCode = http.StatusOK
return res, nil
case fmt.Sprintf("/constellation/v1/attestation/%s/%s.sig", f.attestation.String(), f.olderDate):
res := &http.Response{}
res.Body = io.NopCloser(bytes.NewReader([]byte("null")))
res.StatusCode = http.StatusOK
return res, nil
}
return nil, errors.New("no endpoint found")
}
type stubVerifier struct{}
func (s stubVerifier) VerifySignature(_, _ []byte) error {
return nil
}

View file

@ -0,0 +1,127 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package attestationconfigapi
import (
"encoding/json"
"fmt"
"path"
"sort"
"strings"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
)
// AttestationURLPath is the URL path to the attestation versions.
const AttestationURLPath = "constellation/v1/attestation"
// SEVSNPVersion tracks the latest version of each component for SEV-SNP.
type SEVSNPVersion struct {
// Bootloader is the latest version of the SEV-SNP bootloader.
Bootloader uint8 `json:"bootloader"`
// TEE is the latest version of the SEV-SNP TEE.
TEE uint8 `json:"tee"`
// SNP is the latest version of the SEV-SNP SNP.
SNP uint8 `json:"snp"`
// Microcode is the latest version of the SEV-SNP microcode.
Microcode uint8 `json:"microcode"`
}
// TDXVersion tracks the latest version of each component for TDX.
type TDXVersion struct {
// QESVN is the latest QE security version number.
QESVN uint16 `json:"qeSVN"`
// PCESVN is the latest PCE security version number.
PCESVN uint16 `json:"pceSVN"`
// TEETCBSVN are the latest component-wise security version numbers for the TEE.
TEETCBSVN [16]byte `json:"teeTCBSVN"`
// QEVendorID is the latest QE vendor ID.
QEVendorID [16]byte `json:"qeVendorID"`
// XFAM is the latest XFAM field.
XFAM [8]byte `json:"xfam"`
}
// Entry is the request to get the version information of the specific version in the config api.
//
// TODO: Because variant is not part of the marshalled JSON, fetcher and client methods need to fill the variant property.
// In API v2 we should embed the variant in the object and remove some code from fetcher & client.
// That would remove the possibility of some fetcher/client code forgetting to set the variant.
type Entry struct {
Version string `json:"-"`
Variant variant.Variant `json:"-"`
SEVSNPVersion
TDXVersion
}
// JSONPath returns the path to the JSON file for the request to the config api.
func (i Entry) JSONPath() string {
return path.Join(AttestationURLPath, i.Variant.String(), i.Version)
}
// ValidateRequest validates the request.
func (i Entry) ValidateRequest() error {
if !strings.HasSuffix(i.Version, ".json") {
return fmt.Errorf("version has no .json suffix")
}
return nil
}
// Validate is a No-Op at the moment.
func (i Entry) Validate() error {
return nil
}
// List is the request to retrieve of all versions in the API for one attestation variant.
//
// TODO: Because variant is not part of the marshalled JSON, fetcher and client methods need to fill the variant property.
// In API v2 we should embed the variant in the object and remove some code from fetcher & client.
// That would remove the possibility of some fetcher/client code forgetting to set the variant.
type List struct {
Variant variant.Variant
List []string
}
// MarshalJSON marshals the i's list property to JSON.
func (i List) MarshalJSON() ([]byte, error) {
return json.Marshal(i.List)
}
// UnmarshalJSON unmarshals a list of strings into i's list property.
func (i *List) UnmarshalJSON(data []byte) error {
return json.Unmarshal(data, &i.List)
}
// JSONPath returns the path to the JSON file for the request to the config api.
func (i List) JSONPath() string {
return path.Join(AttestationURLPath, i.Variant.String(), "list")
}
// ValidateRequest is a NoOp as there is no input.
func (i List) ValidateRequest() error {
return nil
}
// SortReverse sorts the list of versions in reverse order.
func (i *List) SortReverse() {
sort.Sort(sort.Reverse(sort.StringSlice(i.List)))
}
// AddVersion adds new to i's list and sorts the element in descending order.
func (i *List) AddVersion(new string) {
i.List = append(i.List, new)
i.List = variant.RemoveDuplicate(i.List)
i.SortReverse()
}
// Validate validates the response.
func (i List) Validate() error {
if len(i.List) < 1 {
return fmt.Errorf("no versions found in /list")
}
return nil
}

View file

@ -0,0 +1,77 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package attestationconfigapi
import (
"testing"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestVersionListMarshalUnmarshalJSON(t *testing.T) {
tests := map[string]struct {
input List
output List
wantDiff bool
}{
"success": {
input: List{List: []string{"v1", "v2"}},
output: List{List: []string{"v1", "v2"}},
},
"variant is lost": {
input: List{List: []string{"v1", "v2"}, Variant: variant.AzureSEVSNP{}},
output: List{List: []string{"v1", "v2"}},
},
"wrong order": {
input: List{List: []string{"v1", "v2"}},
output: List{List: []string{"v2", "v1"}},
wantDiff: true,
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
inputRaw, err := tc.input.MarshalJSON()
require.NoError(t, err)
var actual List
err = actual.UnmarshalJSON(inputRaw)
require.NoError(t, err)
if tc.wantDiff {
assert.NotEqual(t, tc.output, actual, "Objects are equal, expected unequal")
} else {
assert.Equal(t, tc.output, actual, "Objects are not equal, expected equal")
}
})
}
}
func TestVersionListAddVersion(t *testing.T) {
tests := map[string]struct {
versions []string
new string
expected []string
}{
"success": {
versions: []string{"v1", "v2"},
new: "v3",
expected: []string{"v3", "v2", "v1"},
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
v := List{List: tc.versions}
v.AddVersion(tc.new)
assert.Equal(t, tc.expected, v.List)
})
}
}