Add release trigger to make image versions available via CDN

.github/workflows/on-release.yml

@@ -0,0 +1,39 @@
+name: Make updated OS images available on release
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Semantic version tag of the release (vX.Y.Z)."
+        required: true
+
+jobs:
+  update:
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # tag=v3.1.0
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: Setup Go environment
+        uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f # tag=v3.3.1
+        with:
+          go-version: "1.19.3"
+          cache: true
+
+      - name: Login to AWS
+        uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # v1.7.0
+        with:
+          role-to-assume: arn:aws:iam::795746500882:role/GithubAddReleaseVersion
+          aws-region: eu-central-1
+
+      - name: Update OS images
+        run: |
+          go run main.go --version "${{ github.event.release.tag_name }}${{ github.event.inputs.tag }}"
+        working-directory: hack/add-version

hack/add-version/main.go

@@ -0,0 +1,322 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+// add-version adds a new constellation release version to the list of available versions.
+// It is meant to be run by the CI pipeline to make new versions available / discoverable.
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"path"
+	"sort"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/cloudfront"
+	cftypes "github.com/aws/aws-sdk-go-v2/service/cloudfront/types"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/edgelesssys/constellation/v2/internal/constants"
+	"github.com/edgelesssys/constellation/v2/internal/logger"
+	"github.com/edgelesssys/constellation/v2/internal/update"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"golang.org/x/mod/semver"
+)
+
+var errVersionListMissing = errors.New("version list does not exist")
+
+const (
+	stream                       = "stable"
+	imageKind                    = "image"
+	defaultRegion                = "eu-central-1"
+	defaultBucket                = "cdn-constellation-backend"
+	defaultDistributionID        = "E1H77EZTHC3NE4"
+	maxCacheInvalidationWaitTime = 5 * time.Minute
+)
+
+func main() {
+	version := flag.String("version", "", "Version to add (format: \"v1.2.3\")")
+	region := flag.String("region", defaultRegion, "AWS region")
+	bucket := flag.String("bucket", defaultBucket, "S3 bucket")
+	distributionID := flag.String("distribution-id", defaultDistributionID, "cloudfront distribution id")
+	flag.Parse()
+
+	log := logger.New(logger.JSONLog, zapcore.InfoLevel)
+	if err := validateVersion(*version); err != nil {
+		log.With(zap.Error(err)).Fatalf("Invalid version")
+	}
+	major := semver.Major(*version)
+	minor := semver.MajorMinor(*version)
+
+	ctx := context.Background()
+
+	updateFetcher := update.New()
+	versionManager, err := newVersionManager(ctx, *region, *bucket, *distributionID)
+	if err != nil {
+		log.With(zap.Error(err)).Fatalf("Failed to create version uploader")
+	}
+
+	// ensure minor version exists in list for base major version
+	minorVersions, err := versionManager.getMinorVersions(ctx, *version)
+	if err != nil {
+		if !errors.Is(err, errVersionListMissing) {
+			log.With(zap.Error(err)).Fatalf("Failed to get minor versions")
+		}
+		log.Infof("Version list for minor versions under %q does not exist. Creating new list.", major)
+		minorVersions = &update.VersionsList{
+			Stream:      stream,
+			Granularity: "major",
+			Base:        major,
+			Kind:        imageKind,
+			Versions:    []string{},
+		}
+	}
+	if minorVersions.Contains(minor) {
+		log.Infof("Version %q already exists in list %v.", minor, minorVersions.Versions)
+	} else {
+		if err := versionManager.addMinorVersion(ctx, *version, minorVersions); err != nil {
+			log.With(zap.Error(err)).Fatalf("Failed to add minor version")
+		}
+		log.Infof("Added %q to list.", minor)
+	}
+
+	// ensure patch version exists in list for base minor version
+	patchVersions, err := versionManager.getPatchVersions(ctx, *version)
+	if err != nil {
+		if !errors.Is(err, errVersionListMissing) {
+			log.With(zap.Error(err)).Fatalf("Failed to get patch versions")
+		}
+		log.Infof("Version list for patch versions under %q does not exist. Creating new list.", minor)
+		patchVersions = &update.VersionsList{
+			Stream:      stream,
+			Granularity: "minor",
+			Base:        minor,
+			Kind:        imageKind,
+			Versions:    []string{},
+		}
+	}
+	if patchVersions.Contains(*version) {
+		log.Infof("Version %q already exists in list %v.", *version, patchVersions.Versions)
+	} else {
+		if err := versionManager.addPatchVersion(ctx, *version, patchVersions); err != nil {
+			log.With(zap.Error(err)).Fatalf("Failed to add patch version")
+		}
+		log.Infof("Added %q to list.", *version)
+	}
+
+	log.Infof("Successfully added version %q at the following URLs:", *version)
+	log.Infof("major to minor url: %s", versionURL("major", major))
+	log.Infof("minor to patch url: %s", versionURL("minor", minor))
+
+	log.Infof("Waiting for cache invalidation.")
+	if err := versionManager.invalidateCaches(ctx, *version); err != nil {
+		log.With(zap.Error(err)).Fatalf("Failed to invalidate caches")
+	}
+
+	sawAddedVersions := true
+	if err := ensureMinorVersionExists(ctx, updateFetcher, *version); err != nil {
+		sawAddedVersions = false
+		log.Warnf("Failed to ensure minor version exists: %v. This may be resolved by waiting.", err)
+	}
+
+	if err := ensurePatchVersionExists(ctx, updateFetcher, *version); err != nil {
+		sawAddedVersions = false
+		log.Warnf("Failed to ensure patch version exists: %v. This may be resolved by waiting.", err)
+	}
+
+	if sawAddedVersions {
+		log.Infof("Versions are available via API.")
+	}
+}
+
+func validateVersion(version string) error {
+	if !semver.IsValid(version) {
+		return fmt.Errorf("version %q is not a valid semantic version", version)
+	}
+	if semver.Canonical(version) != version {
+		return fmt.Errorf("version %q is not a canonical semantic version", version)
+	}
+	return nil
+}
+
+func ensureMinorVersionExists(ctx context.Context, fetcher *update.VersionsFetcher, version string) error {
+	major := semver.Major(version)
+	minor := semver.MajorMinor(version)
+	existingMinorVersions, err := fetcher.MinorVersionsOf(ctx, stream, major, imageKind)
+	if err != nil {
+		return err
+	}
+	if !existingMinorVersions.Contains(minor) {
+		return errors.New("minor version does not exist")
+	}
+	return nil
+}
+
+func ensurePatchVersionExists(ctx context.Context, fetcher *update.VersionsFetcher, version string) error {
+	minor := semver.MajorMinor(version)
+	patch := semver.Canonical(version)
+	existingPatchVersions, err := fetcher.PatchVersionsOf(ctx, stream, minor, imageKind)
+	if err != nil {
+		return err
+	}
+	if !existingPatchVersions.Contains(patch) {
+		return errors.New("patch version does not exist")
+	}
+	return nil
+}
+
+type versionManager struct {
+	config         aws.Config
+	cloudfrontc    *cloudfront.Client
+	s3c            *s3.Client
+	uploader       *s3manager.Uploader
+	bucket         string
+	distributionID string
+}
+
+func newVersionManager(ctx context.Context, region, bucket, distributionID string) (*versionManager, error) {
+	cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region))
+	if err != nil {
+		return nil, err
+	}
+	cloudfrontc := cloudfront.NewFromConfig(cfg)
+	s3c := s3.NewFromConfig(cfg)
+	uploader := s3manager.NewUploader(s3c)
+	return &versionManager{
+		config:         cfg,
+		cloudfrontc:    cloudfrontc,
+		s3c:            s3c,
+		uploader:       uploader,
+		bucket:         bucket,
+		distributionID: distributionID,
+	}, nil
+}
+
+func (m *versionManager) getMinorVersions(ctx context.Context, version string) (*update.VersionsList, error) {
+	baseVersion := semver.Major(version)
+	return m.getVersions(ctx, baseVersion)
+}
+
+func (m *versionManager) getPatchVersions(ctx context.Context, version string) (*update.VersionsList, error) {
+	baseVersion := semver.MajorMinor(version)
+	return m.getVersions(ctx, baseVersion)
+}
+
+func (m *versionManager) addMinorVersion(ctx context.Context, version string, minorVersions *update.VersionsList) error {
+	baseVersion := semver.Major(version)
+	minorVersion := semver.MajorMinor(version)
+	return m.addVersion(ctx, baseVersion, minorVersion, minorVersions)
+}
+
+func (m *versionManager) addPatchVersion(ctx context.Context, version string, patchVersions *update.VersionsList) error {
+	baseVersion := semver.MajorMinor(version)
+	patchVersion := semver.Canonical(version)
+	return m.addVersion(ctx, baseVersion, patchVersion, patchVersions)
+}
+
+func (m *versionManager) getVersions(ctx context.Context, baseVersion string) (*update.VersionsList, error) {
+	granularity, err := granularityFromVersion(baseVersion)
+	if err != nil {
+		return nil, err
+	}
+	out, err := m.s3c.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(m.bucket),
+		Key:    aws.String(versionJSONPath(granularity, baseVersion)),
+	})
+	if err != nil {
+		var nosuchkey *s3types.NoSuchKey
+		if errors.As(err, &nosuchkey) {
+			return nil, errVersionListMissing
+		}
+		return nil, err
+	}
+	defer out.Body.Close()
+	var versions update.VersionsList
+	if err := json.NewDecoder(out.Body).Decode(&versions); err != nil {
+		return nil, err
+	}
+	return &versions, nil
+}
+
+func (m *versionManager) addVersion(ctx context.Context, baseVersion, version string, list *update.VersionsList) error {
+	granularity, err := granularityFromVersion(baseVersion)
+	if err != nil {
+		return err
+	}
+	list.Versions = append(list.Versions, version)
+	sort.Strings(list.Versions)
+	if err := list.Validate(); err != nil {
+		return err
+	}
+	rawList, err := json.Marshal(list)
+	if err != nil {
+		return err
+	}
+	_, err = m.uploader.Upload(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(m.bucket),
+		Key:    aws.String(versionJSONPath(granularity, baseVersion)),
+		Body:   bytes.NewBuffer(rawList),
+	})
+	return err
+}
+
+func (m *versionManager) invalidateCaches(ctx context.Context, version string) error {
+	major := semver.Major(version)
+	minor := semver.MajorMinor(version)
+	invalidation, err := m.cloudfrontc.CreateInvalidation(ctx, &cloudfront.CreateInvalidationInput{
+		DistributionId: aws.String(m.distributionID),
+		InvalidationBatch: &cftypes.InvalidationBatch{
+			CallerReference: aws.String(fmt.Sprintf("%d", time.Now().Unix())),
+			Paths: &cftypes.Paths{
+				Quantity: aws.Int32(2),
+				Items: []string{
+					"/" + versionJSONPath("major", major),
+					"/" + versionJSONPath("minor", minor),
+				},
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+	waiter := cloudfront.NewInvalidationCompletedWaiter(m.cloudfrontc)
+	if err := waiter.Wait(ctx, &cloudfront.GetInvalidationInput{
+		DistributionId: aws.String(m.distributionID),
+		Id:             invalidation.Invalidation.Id,
+	}, maxCacheInvalidationWaitTime); err != nil {
+		return err
+	}
+	return nil
+}
+
+func granularityFromVersion(version string) (string, error) {
+	switch {
+	case semver.Major(version) == version:
+		return "major", nil
+	case semver.MajorMinor(version) == version:
+		return "minor", nil
+	case semver.Canonical(version) == version:
+		return "patch", nil
+	default:
+		return "", fmt.Errorf("invalid version %q", version)
+	}
+}
+
+func versionJSONPath(granularity, base string) string {
+	return path.Join("constellation/v1/updates", stream, granularity, base, imageKind+".json")
+}
+
+func versionURL(granularity, base string) string {
+	return constants.CDNRepositoryURL + "/" + versionJSONPath(granularity, base)
+}

hack/go.mod

@@ -80,22 +80,24 @@ require (
 	github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
-	github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.17.1
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.18.3
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.42
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/cloudfront v1.21.0
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.73.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.20 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.19 // indirect
 	github.com/aws/aws-sdk-go-v2/service/kms v1.18.18 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.29.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.29.4
 	github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect

hack/go.sum

@@ -207,6 +207,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.13.3 h1:ur+FHdp4NbVIv/49bUjBW+FE7e57
 github.com/aws/aws-sdk-go-v2/credentials v1.13.3/go.mod h1:/rOMmqYBcFfNbRPU0iN9IgGqD5+V2yp3iWNmIlz0wI4=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.42 h1:bxgBYvvBh+W1RnNYP4ROXEB8N+HSSucDszfE7Rb+kfU=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.42/go.mod h1:LHOsygMiW/14CkFxdXxvzKyMh3jbk/QfZVaDtCbLkl8=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 h1:oRHDrwCTVT8ZXi4sr9Ld+EXk7N/KGssOr2ygNeojEhw=
@@ -215,6 +217,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 h1:Mza+vlnZr+fPKFKRq/lKGVvM6B/
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26/go.mod h1:Y2OJ+P+MC1u1VKnavT+PshiEuGPyh/7DqxoDNij4/bg=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.16 h1:2EXB7dtGwRYIN3XQ9qwIW504DVbKIw3r89xQnonGdsQ=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.16/go.mod h1:XH+3h395e3WVdd6T2Z3mPxuI+x/HVtdqVOREkTiyubs=
+github.com/aws/aws-sdk-go-v2/service/cloudfront v1.21.0 h1:3bV+nNw0PPxkHvdRWUogUMtikIgIo2bVj48MvxQvoHA=
+github.com/aws/aws-sdk-go-v2/service/cloudfront v1.21.0/go.mod h1:fV9PzXIIT2xUvf5KXEGToDYxEXcqDyO33SRfv2rAs/8=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.73.0 h1:3AXOhjvPxEMWw5RItV47NRLuzqwlLly5GbS5aB3sXh4=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.73.0/go.mod h1:zul71QqzR4D1a90/5FloZiAnZ1CtuIjVH7R9MP997+A=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10 h1:dpiPHgmFstgkLG07KaYAewvuptq5kvo52xn7tVSrtrQ=

internal/update/update.go

@@ -43,14 +43,14 @@ type VersionsList struct {
 	Versions []string `json:"versions"`
 }
 
-// validate checks if the list is valid.
+// Validate checks if the list is valid.
 // This performs the following checks:
 // - The stream is supported.
 // - The granularity is "major" or "minor".
 // - The kind is supported.
 // - The base version is a valid semantic version that matches the granularity.
 // - All versions in the list are valid semantic versions that are finer-grained than the base version.
-func (l *VersionsList) validate() error {
+func (l *VersionsList) Validate() error {
 	var issues []string
 	if l.Stream != "stable" {
 		issues = append(issues, fmt.Sprintf("stream %q is not supported", l.Stream))
@@ -90,6 +90,16 @@ func (l *VersionsList) validate() error {
 	return nil
 }
 
+// Contains returns true if the list contains the given version.
+func (l *VersionsList) Contains(version string) bool {
+	for _, v := range l.Versions {
+		if v == version {
+			return true
+		}
+	}
+	return false
+}
+
 // VersionsFetcher fetches a list of versions.
 type VersionsFetcher struct {
 	httpc httpc
@@ -122,7 +132,7 @@ func (f *VersionsFetcher) list(ctx context.Context, stream, granularity, base, k
 	if err := json.Unmarshal(raw, &list); err != nil {
 		return nil, fmt.Errorf("decoding versions list: %w", err)
 	}
-	if err := list.validate(); err != nil {
+	if err := list.Validate(); err != nil {
 		return nil, fmt.Errorf("validating versions list: %w", err)
 	}
 	if !f.listMatchesRequest(list, stream, granularity, base, kind) {

internal/update/update_test.go

@@ -102,7 +102,7 @@ func TestValidate(t *testing.T) {
 			if tc.overrideFunc != nil {
 				tc.overrideFunc(list)
 			}
-			err := list.validate()
+			err := list.Validate()
 			if tc.wantErr {
 				assert.Error(err)
 				return