ci: replace add-version through versionsapi cli

Signed-off-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>

.github/workflows/build-os-image.yml

@@ -739,12 +739,11 @@ jobs:
           go-version: "1.19.4"
           cache: true
 
-      - name: Update list of available OS image versions
+      - name: Add version to versionsapi
         if: needs.build-settings.outputs.ref != '-'
-        run: |
+        uses: ./.github/workflows/versionsapi
-          go run main.go \
+        with:
-            --version "${{ needs.build-settings.outputs.imageVersion }}" \
+          ref: ${{ needs.build-settings.outputs.ref }}
-            --stream "${{ inputs.stream }}" \
+          stream: ${{ inputs.stream }}
-            --ref "${{ needs.build-settings.outputs.ref }}" \
+          version: ${{ needs.build-settings.outputs.imageVersion }}
-            --latest
+          add_latest: true
-        working-directory: hack/add-version

.github/workflows/on-release.yml

@@ -37,12 +37,16 @@ jobs:
           role-to-assume: arn:aws:iam::795746500882:role/GithubAddReleaseVersion
           aws-region: eu-central-1
 
-      - name: Update OS images
+      - name: Build versionsapi CLI
-        working-directory: hack/add-version
+        working-directory: internal/versionsapi/cli
+        run: go build -o versionsapi
+
+      - name: Add version to versionsapi
+        working-directory: internal/versionsapi/cli
         run: |
           latest=$([[ "${{ inputs.latest }}" = "true" ]] && echo "--latest" || echo "")
-          go run main.go \
+          ./versionsapi add \
-            --version "${{ github.event.release.tag_name }}${{ github.event.inputs.tag }}" \
-            --stream stable \
             --release \
+            --stream "stable" \
+            --version "${{ github.event.release.tag_name }}${{ github.event.inputs.tag }}" \
             "${latest}"

hack/add-version/main.go

@@ -1,485 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-// add-version adds a new constellation release version to the list of available versions.
-// It is meant to be run by the CI pipeline to make new versions available / discoverable.
-package main
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"flag"
-	"fmt"
-	"path"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	awsconfig "github.com/aws/aws-sdk-go-v2/config"
-	s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
-	"github.com/aws/aws-sdk-go-v2/service/cloudfront"
-	cftypes "github.com/aws/aws-sdk-go-v2/service/cloudfront/types"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/edgelesssys/constellation/v2/internal/constants"
-	"github.com/edgelesssys/constellation/v2/internal/logger"
-	"github.com/edgelesssys/constellation/v2/internal/versionsapi-old"
-	"go.uber.org/zap"
-	"go.uber.org/zap/zapcore"
-	"golang.org/x/mod/semver"
-)
-
-var errVersionListMissing = errors.New("version list does not exist")
-
-const (
-	skipRefStr                   = "-"
-	imageKind                    = "image"
-	defaultRegion                = "eu-central-1"
-	defaultBucket                = "cdn-constellation-backend"
-	defaultDistributionID        = "E1H77EZTHC3NE4"
-	maxCacheInvalidationWaitTime = 5 * time.Minute
-)
-
-func main() {
-	log := logger.New(logger.JSONLog, zapcore.InfoLevel)
-	ctx := context.Background()
-
-	flags := flags{
-		version:        flag.String("version", "", "Version to add (format: \"v1.2.3\")"),
-		stream:         flag.String("stream", "", "Stream to add the version to"),
-		ref:            flag.String("ref", "", "Ref to add the version to"),
-		release:        flag.Bool("release", false, "Whether the version is a release"),
-		latest:         flag.Bool("latest", false, "Whether to set this version as the new latest version"),
-		dryRun:         flag.Bool("dryrun", false, "Whether to run in dry-run mode (no changes are made)"),
-		region:         flag.String("region", defaultRegion, "AWS region"),
-		bucket:         flag.String("bucket", defaultBucket, "S3 bucket"),
-		distributionID: flag.String("distribution-id", defaultDistributionID, "cloudfront distribution id"),
-	}
-	flag.Parse()
-	if err := flags.validate(); err != nil {
-		log.With(zap.Error(err)).Fatalf("Invalid flags")
-	}
-
-	updateFetcher := versionsapi.New()
-	versionManager, err := newVersionManager(ctx, *flags.region, *flags.bucket, *flags.distributionID, *flags.dryRun, log)
-	if err != nil {
-		log.With(zap.Error(err)).Fatalf("Failed to create version uploader")
-	}
-
-	ver := version{
-		versionStr: *flags.version,
-		stream:     *flags.stream,
-		ref:        *flags.ref,
-	}
-
-	if err := ensureMinorVersion(ctx, versionManager, ver, log); err != nil {
-		log.With(zap.Error(err)).Fatalf("Failed to ensure minor version")
-	}
-
-	added, err := ensurePatchVersion(ctx, versionManager, ver, log)
-	if err != nil {
-		log.With(zap.Error(err)).Fatalf("Failed to ensure patch version")
-	}
-
-	if added && *flags.latest {
-		if err := versionManager.addLatest(ctx, ver); err != nil {
-			log.With(zap.Error(err)).Fatalf("Failed to update latest version object")
-		}
-		log.Infof("Added %q as latest version.", ver)
-	}
-
-	log.Infof("Major to minor url: %s", ver.URL(granularityMajor))
-	log.Infof("Minor to patch url: %s", ver.URL(granularityMinor))
-
-	if !versionManager.dirty {
-		log.Infof("No changes made, everything up to date.")
-		return
-	}
-	log.Infof("Successfully added version %q", *flags.version)
-
-	log.Infof("Waiting for cache invalidation.")
-	if err := versionManager.invalidateCaches(ctx, ver, *flags.latest); err != nil {
-		log.With(zap.Error(err)).Fatalf("Failed to invalidate caches")
-	}
-
-	waitForCacheUpdate(ctx, updateFetcher, ver, log)
-}
-
-func ensureMinorVersion(ctx context.Context, versionManager *versionManager, ver version, log *logger.Logger) error {
-	minorVerList, err := versionManager.getVersionList(ctx, ver, granularityMajor)
-	log.Debugf("Minor version list: %v", minorVerList)
-	if errors.Is(err, errVersionListMissing) {
-		log.Infof("Version list for minor versions under %q does not exist. Creating new list.", ver.Major())
-		minorVerList = &versionsapi.List{
-			Ref:         ver.Ref(),
-			Stream:      ver.Stream(),
-			Granularity: "major",
-			Base:        ver.Major(),
-			Kind:        imageKind,
-			Versions:    []string{},
-		}
-	} else if err != nil {
-		return fmt.Errorf("failed to list minor versions: %w", err)
-	}
-
-	if minorVerList.Contains(ver.MajorMinor()) {
-		log.Infof("Version %q already exists in list %v.", ver.MajorMinor(), minorVerList.Versions)
-		return nil
-	}
-
-	minorVerList.Versions = append(minorVerList.Versions, ver.MajorMinor())
-	log.Debugf("New minor version list: %v", minorVerList)
-
-	if err := versionManager.updateVersionList(ctx, minorVerList); err != nil {
-		return fmt.Errorf("failed to add minor version: %w", err)
-	}
-
-	log.Infof("Added %q to list.", ver.MajorMinor())
-	return nil
-}
-
-func ensurePatchVersion(ctx context.Context, versionManager *versionManager, ver version, log *logger.Logger) (bool, error) {
-	pathVerList, err := versionManager.getVersionList(ctx, ver, granularityMinor)
-	if errors.Is(err, errVersionListMissing) {
-		log.Infof("Version list for patch versions under %q does not exist. Creating new list.", ver.MajorMinor())
-		pathVerList = &versionsapi.List{
-			Ref:         ver.Ref(),
-			Stream:      ver.Stream(),
-			Granularity: "minor",
-			Base:        ver.MajorMinor(),
-			Kind:        imageKind,
-			Versions:    []string{},
-		}
-	} else if err != nil {
-		return false, fmt.Errorf("failed to get patch versions: %w", err)
-	}
-
-	if pathVerList.Contains(ver.String()) {
-		log.Infof("Version %q already exists in list %v.", ver.String(), pathVerList.Versions)
-		return false, nil
-	}
-
-	pathVerList.Versions = append(pathVerList.Versions, ver.String())
-
-	if err := versionManager.updateVersionList(ctx, pathVerList); err != nil {
-		log.With(zap.Error(err)).Fatalf("Failed to add patch version")
-	}
-
-	log.Infof("Added %q to list.", ver.String())
-	return true, nil
-}
-
-type version struct {
-	versionStr string
-	stream     string
-	ref        string
-}
-
-func (v *version) String() string {
-	return semver.Canonical(v.versionStr)
-}
-
-func (v *version) Major() string {
-	return semver.Major(v.versionStr)
-}
-
-func (v *version) MajorMinor() string {
-	return semver.MajorMinor(v.versionStr)
-}
-
-func (v *version) WithGranularity(gran granularity) string {
-	switch gran {
-	case granularityMajor:
-		return v.Major()
-	case granularityMinor:
-		return v.MajorMinor()
-	default:
-		return ""
-	}
-}
-
-func (v *version) URL(gran granularity) string {
-	return constants.CDNRepositoryURL + "/" + v.JSONPath(gran)
-}
-
-func (v *version) JSONPath(gran granularity) string {
-	return path.Join(constants.CDNAPIPrefix, "ref", v.ref, "stream", v.stream, "versions", gran.String(), v.WithGranularity(gran), imageKind+".json")
-}
-
-func (v *version) Stream() string {
-	return v.stream
-}
-
-func (v *version) Ref() string {
-	return v.ref
-}
-
-type flags struct {
-	version        *string
-	stream         *string
-	ref            *string
-	release        *bool
-	latest         *bool
-	dryRun         *bool
-	region         *string
-	bucket         *string
-	distributionID *string
-}
-
-func (f *flags) validate() error {
-	if err := validateVersion(*f.version); err != nil {
-		return err
-	}
-
-	if *f.ref == "" && !*f.release {
-		if !*f.release {
-			return fmt.Errorf("branch flag must be set for non-release versions")
-		}
-	}
-
-	if *f.ref != "" && *f.release {
-		return fmt.Errorf("branch flag must not be set for release versions")
-	}
-
-	if *f.release {
-		*f.ref = skipRefStr
-	} else {
-		*f.latest = true // always set latest for non-release versions
-	}
-
-	ref := versionsapi.CanonicalRef(*f.ref)
-	if !versionsapi.IsValidRef(ref) {
-		return fmt.Errorf("invalid ref %q", *f.ref)
-	}
-	*f.ref = ref
-
-	if !versionsapi.IsValidStream(*f.ref, *f.stream) {
-		return fmt.Errorf("invalid stream %q for ref %q", *f.stream, *f.ref)
-	}
-
-	return nil
-}
-
-func validateVersion(version string) error {
-	if !semver.IsValid(version) {
-		return fmt.Errorf("version %q is not a valid semantic version", version)
-	}
-	if semver.Canonical(version) != version {
-		return fmt.Errorf("version %q is not a canonical semantic version", version)
-	}
-	return nil
-}
-
-func ensureMinorVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher, ver version) error {
-	existingMinorVersions, err := fetcher.MinorVersionsOf(ctx, ver.Ref(), ver.Stream(), ver.Major(), imageKind)
-	if err != nil {
-		return err
-	}
-	if !existingMinorVersions.Contains(ver.MajorMinor()) {
-		return errors.New("minor version does not exist")
-	}
-	return nil
-}
-
-func ensurePatchVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher, ver version) error {
-	existingPatchVersions, err := fetcher.PatchVersionsOf(ctx, ver.Ref(), ver.Stream(), ver.MajorMinor(), imageKind)
-	if err != nil {
-		return err
-	}
-	if !existingPatchVersions.Contains(ver.String()) {
-		return errors.New("patch version does not exist")
-	}
-	return nil
-}
-
-type versionManager struct {
-	config         aws.Config
-	cloudfrontc    *cloudfront.Client
-	s3c            *s3.Client
-	uploader       *s3manager.Uploader
-	bucket         string
-	distributionID string
-	dirty          bool // manager gets dirty on write
-	dryRun         bool // no write operations
-	log            *logger.Logger
-}
-
-func newVersionManager(ctx context.Context, region, bucket, distributionID string, dryRun bool, log *logger.Logger) (*versionManager, error) {
-	cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region))
-	if err != nil {
-		return nil, err
-	}
-	cloudfrontc := cloudfront.NewFromConfig(cfg)
-	s3c := s3.NewFromConfig(cfg)
-	uploader := s3manager.NewUploader(s3c)
-	return &versionManager{
-		config:         cfg,
-		cloudfrontc:    cloudfrontc,
-		s3c:            s3c,
-		uploader:       uploader,
-		bucket:         bucket,
-		distributionID: distributionID,
-		dryRun:         dryRun,
-		log:            log,
-	}, nil
-}
-
-func (m *versionManager) getVersionList(ctx context.Context, ver version, gran granularity) (*versionsapi.List, error) {
-	in := &s3.GetObjectInput{
-		Bucket: aws.String(m.bucket),
-		Key:    aws.String(ver.JSONPath(gran)),
-	}
-	out, err := m.s3c.GetObject(ctx, in)
-	var noSuchkey *s3types.NoSuchKey
-	if errors.As(err, &noSuchkey) {
-		return nil, errVersionListMissing
-	} else if err != nil {
-		return nil, err
-	}
-	defer out.Body.Close()
-
-	var list versionsapi.List
-	if err := json.NewDecoder(out.Body).Decode(&list); err != nil {
-		return nil, err
-	}
-
-	return &list, nil
-}
-
-func (m *versionManager) updateVersionList(ctx context.Context, list *versionsapi.List) error {
-	semver.Sort(list.Versions)
-	if err := list.Validate(); err != nil {
-		return err
-	}
-
-	rawList, err := json.Marshal(list)
-	if err != nil {
-		return err
-	}
-
-	in := &s3.PutObjectInput{
-		Bucket: aws.String(m.bucket),
-		Key:    aws.String(list.JSONPath()),
-		Body:   bytes.NewBuffer(rawList),
-	}
-
-	if m.dryRun {
-		m.log.Infof("dryRun: s3 put object {Bucket: %v, Key: %v, Body: %v", m.bucket, list.JSONPath(), string(rawList))
-		return nil
-	}
-
-	m.dirty = true
-
-	_, err = m.uploader.Upload(ctx, in)
-
-	return err
-}
-
-func (m *versionManager) addLatest(ctx context.Context, ver version) error {
-	latest := &versionsapi.Latest{
-		Ref:     ver.Ref(),
-		Stream:  ver.Stream(),
-		Kind:    imageKind,
-		Version: ver.String(),
-	}
-	if err := latest.Validate(); err != nil {
-		return err
-	}
-
-	rawLatest, err := json.Marshal(latest)
-	if err != nil {
-		return err
-	}
-
-	in := &s3.PutObjectInput{
-		Bucket: aws.String(m.bucket),
-		Key:    aws.String(latest.JSONPath()),
-		Body:   bytes.NewBuffer(rawLatest),
-	}
-
-	if m.dryRun {
-		m.log.Infof("dryRun: s3 put object {Bucket: %v, Key: %v, Body: %v", m.bucket, latest.JSONPath(), string(rawLatest))
-		return nil
-	}
-
-	m.dirty = true
-
-	_, err = m.uploader.Upload(ctx, in)
-
-	return err
-}
-
-func (m *versionManager) invalidateCaches(ctx context.Context, ver version, latest bool) error {
-	invalidIn := &cloudfront.CreateInvalidationInput{
-		DistributionId: aws.String(m.distributionID),
-		InvalidationBatch: &cftypes.InvalidationBatch{
-			CallerReference: aws.String(fmt.Sprintf("%d", time.Now().Unix())),
-			Paths: &cftypes.Paths{
-				Quantity: aws.Int32(2),
-				Items: []string{
-					"/" + ver.URL(granularityMajor),
-					"/" + ver.URL(granularityMinor),
-				},
-			},
-		},
-	}
-	if latest {
-		invalidIn.InvalidationBatch.Paths.Quantity = aws.Int32(3)
-		path := path.Join("ref", ver.Ref(), "stream", ver.Stream(), "versions/latest/image.json")
-		invalidIn.InvalidationBatch.Paths.Items = append(invalidIn.InvalidationBatch.Paths.Items, "/"+path)
-	}
-	invalidation, err := m.cloudfrontc.CreateInvalidation(ctx, invalidIn)
-	if err != nil {
-		return err
-	}
-
-	waiter := cloudfront.NewInvalidationCompletedWaiter(m.cloudfrontc)
-	waitIn := &cloudfront.GetInvalidationInput{
-		DistributionId: aws.String(m.distributionID),
-		Id:             invalidation.Invalidation.Id,
-	}
-	if err := waiter.Wait(ctx, waitIn, maxCacheInvalidationWaitTime); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func waitForCacheUpdate(ctx context.Context, updateFetcher *versionsapi.Fetcher, ver version, log *logger.Logger) {
-	sawAddedVersions := true
-	if err := ensureMinorVersionExists(ctx, updateFetcher, ver); err != nil {
-		sawAddedVersions = false
-		log.Warnf("Failed to ensure minor version exists: %v. This may be resolved by waiting.", err)
-	}
-
-	if err := ensurePatchVersionExists(ctx, updateFetcher, ver); err != nil {
-		sawAddedVersions = false
-		log.Warnf("Failed to ensure patch version exists: %v. This may be resolved by waiting.", err)
-	}
-
-	if sawAddedVersions {
-		log.Infof("Versions are available via API.")
-	}
-}
-
-type granularity int
-
-const (
-	granularityMajor granularity = iota
-	granularityMinor
-)
-
-func (g granularity) String() string {
-	switch g {
-	case granularityMajor:
-		return "major"
-	case granularityMinor:
-		return "minor"
-	default:
-		return "unknown"
-	}
-}

hack/go.mod

@@ -79,24 +79,22 @@ require (
 	github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
-	github.com/aws/aws-sdk-go-v2 v1.17.3
+	github.com/aws/aws-sdk-go-v2 v1.17.3 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.18.7
+	github.com/aws/aws-sdk-go-v2/config v1.18.7 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.7 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 // indirect
-	github.com/aws/aws-sdk-go-v2/service/cloudfront v1.22.2
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.22 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21 // indirect
 	github.com/aws/aws-sdk-go-v2/service/kms v1.19.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 // indirect

hack/go.sum

@@ -209,8 +209,6 @@ github.com/aws/aws-sdk-go-v2/credentials v1.13.7 h1:qUUcNS5Z1092XBFT66IJM7mYkMwg
 github.com/aws/aws-sdk-go-v2/credentials v1.13.7/go.mod h1:AdCcbZXHQCjJh6NaH3pFaw8LUeBFn5+88BZGMVGuBT8=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46 h1:OCX1pQ4pcqhsDV7B92HzdLWjHWOQsILvjLinpaUWhcc=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46/go.mod h1:MxCBOcyNXGJRvfpPiH+L6n/BF9zbowthGSUZdDvQF/c=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE=
@@ -219,8 +217,6 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 h1:H/mF2LNWwX00lD6FlYfKpLLZgUW7oIzCBkig78x4Xok=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18/go.mod h1:T2Ku+STrYQ1zIkL1wMvj8P3wWQaaCMKNdz70MT2FLfE=
-github.com/aws/aws-sdk-go-v2/service/cloudfront v1.22.2 h1:XP88fE1Y8a5308lYzmTnC0be6prtaKAXc6qlD1e4YIE=
-github.com/aws/aws-sdk-go-v2/service/cloudfront v1.22.2/go.mod h1:xUOmvPrMKmH94stXswKsGSkL02vMpNU+rTG+eIzFfNQ=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 h1:m6HYlpZlTWb9vHuuRHpWRieqPHWlS0mvQ90OJNrG/Nk=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0/go.mod h1:mV0E7631M1eXdB+tlGFIw6JxfsC7Pz7+7Aw15oLVhZw=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=

internal/versionsapi-old/versionsapi.go

@@ -1,278 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-package versionsapi
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"path"
-	"regexp"
-	"strings"
-
-	"github.com/edgelesssys/constellation/v2/internal/constants"
-	"golang.org/x/mod/semver"
-)
-
-// List represents a list of versions for a kind of resource.
-// It has a granularity of either "major" or "minor".
-//
-// For example, a List with granularity "major" could contain
-// the base version "v1" and a list of minor versions "v1.0", "v1.1", "v1.2" etc.
-// A List with granularity "minor" could contain the base version
-// "v1.0" and a list of patch versions "v1.0.0", "v1.0.1", "v1.0.2" etc.
-type List struct {
-	// Ref is the branch name the list belongs to.
-	Ref string `json:"ref,omitempty"`
-	// Stream is the update stream of the list.
-	Stream string `json:"stream,omitempty"`
-	// Granularity is the granularity of the base version of this list.
-	// It can be either "major" or "minor".
-	Granularity string `json:"granularity,omitempty"`
-	// Base is the base version of the list.
-	// Every version in the list is a finer-grained version of this base version.
-	Base string `json:"base,omitempty"`
-	// Kind is the kind of resource this list is for.
-	Kind string `json:"kind,omitempty"`
-	// Versions is a list of all versions in this list.
-	Versions []string `json:"versions,omitempty"`
-}
-
-// Validate checks if the list is valid.
-// This performs the following checks:
-// - The ref is set.
-// - The stream is supported.
-// - The granularity is "major" or "minor".
-// - The kind is supported.
-// - The base version is a valid semantic version that matches the granularity.
-// - All versions in the list are valid semantic versions that are finer-grained than the base version.
-func (l *List) Validate() error {
-	var issues []string
-	if !IsValidRef(l.Ref) {
-		issues = append(issues, "ref is empty")
-	}
-	if !IsValidStream(l.Ref, l.Stream) {
-		issues = append(issues, fmt.Sprintf("stream %q is not supported on ref %q", l.Stream, l.Ref))
-	}
-	if l.Granularity != "major" && l.Granularity != "minor" {
-		issues = append(issues, fmt.Sprintf("granularity %q is not supported", l.Granularity))
-	}
-	if l.Kind != "image" {
-		issues = append(issues, fmt.Sprintf("kind %q is not supported", l.Kind))
-	}
-	if !semver.IsValid(l.Base) {
-		issues = append(issues, fmt.Sprintf("base version %q is not a valid semantic version", l.Base))
-	}
-	var normalizeFunc func(string) string
-	switch l.Granularity {
-	case "major":
-		normalizeFunc = semver.Major
-	case "minor":
-		normalizeFunc = semver.MajorMinor
-	default:
-		normalizeFunc = func(s string) string { return s }
-	}
-	if normalizeFunc(l.Base) != l.Base {
-		issues = append(issues, fmt.Sprintf("base version %q is not a %v version", l.Base, l.Granularity))
-	}
-	for _, ver := range l.Versions {
-		if !semver.IsValid(ver) {
-			issues = append(issues, fmt.Sprintf("version %q in list is not a valid semantic version", ver))
-		}
-		if normalizeFunc(ver) != l.Base {
-			issues = append(issues, fmt.Sprintf("version %q in list is not a finer-grained version of base version %q", ver, l.Base))
-		}
-	}
-	if len(issues) > 0 {
-		return fmt.Errorf("version list is invalid:\n%s", strings.Join(issues, "\n"))
-	}
-	return nil
-}
-
-// JSONPath returns the S3 JSON path for this object.
-func (l *List) JSONPath() string {
-	return path.Join(constants.CDNAPIPrefix, "ref", l.Ref, "stream", l.Stream, "versions", l.Granularity, l.Base, l.Kind+".json")
-}
-
-// Latest is the latest version of a kind of resource.
-type Latest struct {
-	// Ref is the branch name this latest version belongs to.
-	Ref string `json:"ref,omitempty"`
-	// Stream is stream name this latest version belongs to.
-	Stream string `json:"stream,omitempty"`
-	// Kind is the kind of resource this latest version is for.
-	Kind string `json:"kind,omitempty"`
-	// Version is the latest version for this ref, stream and kind.
-	Version string `json:"version,omitempty"`
-}
-
-// Validate checks if this latest version is valid.
-func (l *Latest) Validate() error {
-	var issues []string
-	if !IsValidRef(l.Ref) {
-		issues = append(issues, "ref is empty")
-	}
-	if !IsValidStream(l.Ref, l.Stream) {
-		issues = append(issues, fmt.Sprintf("stream %q is not supported on ref %q", l.Stream, l.Ref))
-	}
-	if l.Kind != "image" {
-		issues = append(issues, fmt.Sprintf("kind %q is not supported", l.Kind))
-	}
-	if !semver.IsValid(l.Version) {
-		issues = append(issues, fmt.Sprintf("version %q is not a valid semantic version", l.Version))
-	}
-	if len(issues) > 0 {
-		return fmt.Errorf("latest version is invalid:\n%s", strings.Join(issues, "\n"))
-	}
-	return nil
-}
-
-// JSONPath returns the S3 JSON path for this object.
-func (l *Latest) JSONPath() string {
-	return path.Join(constants.CDNAPIPrefix, "ref", l.Ref, "stream", l.Stream, "versions", "latest", l.Kind+".json")
-}
-
-// Contains returns true if the list contains the given version.
-func (l *List) Contains(version string) bool {
-	for _, v := range l.Versions {
-		if v == version {
-			return true
-		}
-	}
-	return false
-}
-
-// Fetcher fetches a list of versions.
-type Fetcher struct {
-	httpc httpc
-}
-
-// New returns a new VersionsFetcher.
-func New() *Fetcher {
-	return &Fetcher{
-		httpc: http.DefaultClient,
-	}
-}
-
-// MinorVersionsOf fetches the list of minor versions for a given stream, major version and kind.
-func (f *Fetcher) MinorVersionsOf(ctx context.Context, ref, stream, major, kind string) (*List, error) {
-	return f.list(ctx, ref, stream, "major", major, kind)
-}
-
-// PatchVersionsOf fetches the list of patch versions for a given stream, minor version and kind.
-func (f *Fetcher) PatchVersionsOf(ctx context.Context, ref, stream, minor, kind string) (*List, error) {
-	return f.list(ctx, ref, stream, "minor", minor, kind)
-}
-
-// list fetches the list of versions for a given stream, granularity, base and kind.
-func (f *Fetcher) list(ctx context.Context, ref, stream, granularity, base, kind string) (*List, error) {
-	raw, err := getFromURL(ctx, f.httpc, ref, stream, granularity, base, kind)
-	if err != nil {
-		return nil, fmt.Errorf("fetching versions list: %w", err)
-	}
-	list := &List{}
-	if err := json.Unmarshal(raw, &list); err != nil {
-		return nil, fmt.Errorf("decoding versions list: %w", err)
-	}
-	if err := list.Validate(); err != nil {
-		return nil, fmt.Errorf("validating versions list: %w", err)
-	}
-	if !f.listMatchesRequest(list, stream, granularity, base, kind) {
-		return nil, fmt.Errorf("versions list does not match request")
-	}
-	return list, nil
-}
-
-func (f *Fetcher) listMatchesRequest(list *List, stream, granularity, base, kind string) bool {
-	return list.Stream == stream && list.Granularity == granularity && list.Base == base && list.Kind == kind
-}
-
-// getFromURL fetches the versions list from a URL.
-func getFromURL(ctx context.Context, client httpc, ref, stream, granularity, base, kind string) ([]byte, error) {
-	url, err := url.Parse(constants.CDNRepositoryURL)
-	if err != nil {
-		return nil, fmt.Errorf("parsing image version repository URL: %w", err)
-	}
-	kindFilename := path.Base(kind) + ".json"
-	url.Path = path.Join(constants.CDNAPIPrefix, "ref", ref, "stream", stream, "versions", granularity, base, kindFilename)
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), http.NoBody)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		switch resp.StatusCode {
-		case http.StatusNotFound:
-			return nil, fmt.Errorf("versions list %q does not exist", url.String())
-		default:
-			return nil, fmt.Errorf("unexpected status code %d", resp.StatusCode)
-		}
-	}
-	content, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-	return content, nil
-}
-
-// IsValidStream returns true if the given stream is a valid stream.
-func IsValidStream(ref, stream string) bool {
-	validReleaseStreams := []string{"stable", "console", "debug"}
-	validStreams := []string{"nightly", "console", "debug"}
-
-	if isReleaseRef(ref) {
-		validStreams = validReleaseStreams
-	}
-
-	for _, validStream := range validStreams {
-		if stream == validStream {
-			return true
-		}
-	}
-
-	return false
-}
-
-var notAZ09Regexp = regexp.MustCompile("[^a-zA-Z0-9-]+")
-
-// CanonicalRef returns the canonicalized ref for the given ref.
-func CanonicalRef(ref string) string {
-	return notAZ09Regexp.ReplaceAllString(ref, "-")
-}
-
-// IsValidRef returns true if the given ref is a valid ref.
-func IsValidRef(ref string) bool {
-	if ref == "" {
-		return false
-	}
-
-	if notAZ09Regexp.FindString(ref) != "" {
-		return false
-	}
-
-	if strings.HasPrefix(ref, "refs-heads") {
-		return false
-	}
-
-	return true
-}
-
-func isReleaseRef(ref string) bool {
-	return ref == "-"
-}
-
-type httpc interface {
-	Do(req *http.Request) (*http.Response, error)
-}

internal/versionsapi-old/versionsapi_test.go

@@ -1,347 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-package versionsapi
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"io"
-	"net/http"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/goleak"
-)
-
-func TestMain(m *testing.M) {
-	goleak.VerifyTestMain(m)
-}
-
-func TestValidate(t *testing.T) {
-	testCases := map[string]struct {
-		listFunc     func() *List
-		overrideFunc func(list *List)
-		wantErr      bool
-	}{
-		"valid major list": {
-			listFunc: majorList,
-		},
-		"valid minor list": {
-			listFunc: minorList,
-		},
-		"invalid stream": {
-			listFunc: majorList,
-			overrideFunc: func(list *List) {
-				list.Stream = "invalid"
-			},
-			wantErr: true,
-		},
-		"invalid granularity": {
-			listFunc: majorList,
-			overrideFunc: func(list *List) {
-				list.Granularity = "invalid"
-			},
-			wantErr: true,
-		},
-		"invalid kind": {
-			listFunc: majorList,
-			overrideFunc: func(list *List) {
-				list.Kind = "invalid"
-			},
-			wantErr: true,
-		},
-		"base ver is not semantic version": {
-			listFunc: majorList,
-			overrideFunc: func(list *List) {
-				list.Base = "invalid"
-			},
-			wantErr: true,
-		},
-		"base ver does not reflect major granularity": {
-			listFunc: majorList,
-			overrideFunc: func(list *List) {
-				list.Base = "v1.0"
-			},
-			wantErr: true,
-		},
-		"base ver does not reflect minor granularity": {
-			listFunc: minorList,
-			overrideFunc: func(list *List) {
-				list.Base = "v1"
-			},
-			wantErr: true,
-		},
-		"version in list is not semantic version": {
-			listFunc: majorList,
-			overrideFunc: func(list *List) {
-				list.Versions[0] = "invalid"
-			},
-			wantErr: true,
-		},
-		"version in list is not sub version of base": {
-			listFunc: majorList,
-			overrideFunc: func(list *List) {
-				list.Versions[0] = "v2.1"
-			},
-			wantErr: true,
-		},
-	}
-
-	for name, tc := range testCases {
-		t.Run(name, func(t *testing.T) {
-			assert := assert.New(t)
-			require := require.New(t)
-
-			list := tc.listFunc()
-			if tc.overrideFunc != nil {
-				tc.overrideFunc(list)
-			}
-			err := list.Validate()
-			if tc.wantErr {
-				assert.Error(err)
-				return
-			}
-			require.NoError(err)
-		})
-	}
-}
-
-func TestList(t *testing.T) {
-	majorListJSON, err := json.Marshal(majorList())
-	require.NoError(t, err)
-	minorListJSON, err := json.Marshal(minorList())
-	require.NoError(t, err)
-	inconsistentList := majorList()
-	inconsistentList.Base = "v2"
-	inconsistentListJSON, err := json.Marshal(inconsistentList)
-	require.NoError(t, err)
-	client := newTestClient(func(req *http.Request) *http.Response {
-		switch req.URL.Path {
-		case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/image.json":
-			return &http.Response{
-				StatusCode: http.StatusOK,
-				Body:       io.NopCloser(bytes.NewBuffer(majorListJSON)),
-				Header:     make(http.Header),
-			}
-		case "/constellation/v1/ref/test-ref/stream/nightly/versions/minor/v1.1/image.json":
-			return &http.Response{
-				StatusCode: http.StatusOK,
-				Body:       io.NopCloser(bytes.NewBuffer(minorListJSON)),
-				Header:     make(http.Header),
-			}
-		case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/500.json": // 500 error
-			return &http.Response{
-				StatusCode: http.StatusInternalServerError,
-				Body:       io.NopCloser(bytes.NewBufferString("Server Error.")),
-				Header:     make(http.Header),
-			}
-		case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/nojson.json": // invalid format
-			return &http.Response{
-				StatusCode: http.StatusOK,
-				Body:       io.NopCloser(bytes.NewBufferString("not json")),
-				Header:     make(http.Header),
-			}
-		case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v2/image.json": // inconsistent list
-			return &http.Response{
-				StatusCode: http.StatusOK,
-				Body:       io.NopCloser(bytes.NewBuffer(inconsistentListJSON)),
-				Header:     make(http.Header),
-			}
-		case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v3/image.json": // does not match requested version
-			return &http.Response{
-				StatusCode: http.StatusOK,
-				Body:       io.NopCloser(bytes.NewBuffer(minorListJSON)),
-				Header:     make(http.Header),
-			}
-		}
-		return &http.Response{
-			StatusCode: http.StatusNotFound,
-			Body:       io.NopCloser(bytes.NewBufferString("Not found.")),
-			Header:     make(http.Header),
-		}
-	})
-
-	testCases := map[string]struct {
-		ref, stream, granularity, base, kind string
-		overrideFile                         string
-		wantList                             List
-		wantErr                              bool
-	}{
-		"major list fetched remotely": {
-			wantList: *majorList(),
-		},
-		"minor list fetched remotely": {
-			granularity: "minor",
-			base:        "v1.1",
-			wantList:    *minorList(),
-		},
-		"list does not exist": {
-			stream:  "unknown",
-			wantErr: true,
-		},
-		"unexpected error code": {
-			kind:    "500",
-			wantErr: true,
-		},
-		"invalid json returned": {
-			kind:    "nojson",
-			wantErr: true,
-		},
-		"invalid list returned": {
-			base:    "v2",
-			wantErr: true,
-		},
-		"response does not match request": {
-			base:    "v3",
-			wantErr: true,
-		},
-	}
-
-	for name, tc := range testCases {
-		t.Run(name, func(t *testing.T) {
-			assert := assert.New(t)
-			require := require.New(t)
-
-			ref := "test-ref"
-			stream := "nightly"
-			granularity := "major"
-			base := "v1"
-			kind := "image"
-			if tc.stream != "" {
-				stream = tc.stream
-			}
-			if tc.granularity != "" {
-				granularity = tc.granularity
-			}
-			if tc.base != "" {
-				base = tc.base
-			}
-			if tc.kind != "" {
-				kind = tc.kind
-			}
-
-			fetcher := &Fetcher{
-				httpc: client,
-			}
-			list, err := fetcher.list(context.Background(), ref, stream, granularity, base, kind)
-			if tc.wantErr {
-				assert.Error(err)
-				return
-			}
-			require.NoError(err)
-			assert.Equal(tc.wantList, *list)
-		})
-	}
-}
-
-// roundTripFunc .
-type roundTripFunc func(req *http.Request) *http.Response
-
-// RoundTrip .
-func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
-	return f(req), nil
-}
-
-// newTestClient returns *http.Client with Transport replaced to avoid making real calls.
-func newTestClient(fn roundTripFunc) *http.Client {
-	return &http.Client{
-		Transport: fn,
-	}
-}
-
-func majorList() *List {
-	return &List{
-		Ref:         "test-ref",
-		Stream:      "nightly",
-		Granularity: "major",
-		Base:        "v1",
-		Kind:        "image",
-		Versions: []string{
-			"v1.0", "v1.1", "v1.2",
-		},
-	}
-}
-
-func minorList() *List {
-	return &List{
-		Ref:         "test-ref",
-		Stream:      "nightly",
-		Granularity: "minor",
-		Base:        "v1.1",
-		Kind:        "image",
-		Versions: []string{
-			"v1.1.0", "v1.1.1", "v1.1.2",
-		},
-	}
-}
-
-func TestIsValidRef(t *testing.T) {
-	testCases := map[string]bool{
-		"feat/foo":            false,
-		"feat-foo":            true,
-		"feat$foo":            false,
-		"3234":                true,
-		"feat foo":            false,
-		"refs-heads-feat-foo": false,
-		"":                    false,
-	}
-
-	for ref, want := range testCases {
-		t.Run(ref, func(t *testing.T) {
-			assert := assert.New(t)
-			assert.Equal(want, IsValidRef(ref))
-		})
-	}
-}
-
-func TestCanonicalRef(t *testing.T) {
-	testCases := map[string]string{
-		"feat/foo": "feat-foo",
-		"feat-foo": "feat-foo",
-		"feat$foo": "feat-foo",
-		"3234":     "3234",
-		"feat foo": "feat-foo",
-	}
-
-	for ref, want := range testCases {
-		t.Run(ref, func(t *testing.T) {
-			assert := assert.New(t)
-			assert.Equal(want, CanonicalRef(ref))
-		})
-	}
-}
-
-func TestIsValidStream(t *testing.T) {
-	testCases := []struct {
-		branch string
-		stream string
-		want   bool
-	}{
-		{branch: "-", stream: "stable", want: true},
-		{branch: "-", stream: "debug", want: true},
-		{branch: "-", stream: "nightly", want: false},
-		{branch: "-", stream: "console", want: true},
-		{branch: "main", stream: "stable", want: false},
-		{branch: "main", stream: "debug", want: true},
-		{branch: "main", stream: "nightly", want: true},
-		{branch: "main", stream: "console", want: true},
-		{branch: "foo-branch", stream: "nightly", want: true},
-		{branch: "foo-branch", stream: "console", want: true},
-		{branch: "foo-branch", stream: "debug", want: true},
-		{branch: "foo-branch", stream: "stable", want: false},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.branch+"+"+tc.stream, func(t *testing.T) {
-			assert := assert.New(t)
-
-			assert.Equal(tc.want, IsValidStream(tc.branch, tc.stream))
-		})
-	}
-}