From 3561a168192d95bf506b346713f63f48613041ef Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Fri, 30 Dec 2022 10:12:48 +0100 Subject: [PATCH] ci: replace add-version through versionsapi cli Signed-off-by: Paul Meyer <49727155+katexochen@users.noreply.github.com> --- .github/workflows/build-os-image.yml | 15 +- .github/workflows/on-release.yml | 14 +- hack/add-version/main.go | 485 ------------------- hack/go.mod | 8 +- hack/go.sum | 4 - internal/versionsapi-old/versionsapi.go | 278 ----------- internal/versionsapi-old/versionsapi_test.go | 347 ------------- 7 files changed, 19 insertions(+), 1132 deletions(-) delete mode 100644 hack/add-version/main.go delete mode 100644 internal/versionsapi-old/versionsapi.go delete mode 100644 internal/versionsapi-old/versionsapi_test.go diff --git a/.github/workflows/build-os-image.yml b/.github/workflows/build-os-image.yml index 63c35ed49..f0d20d790 100644 --- a/.github/workflows/build-os-image.yml +++ b/.github/workflows/build-os-image.yml @@ -739,12 +739,11 @@ jobs: go-version: "1.19.4" cache: true - - name: Update list of available OS image versions + - name: Add version to versionsapi if: needs.build-settings.outputs.ref != '-' - run: | - go run main.go \ - --version "${{ needs.build-settings.outputs.imageVersion }}" \ - --stream "${{ inputs.stream }}" \ - --ref "${{ needs.build-settings.outputs.ref }}" \ - --latest - working-directory: hack/add-version + uses: ./.github/workflows/versionsapi + with: + ref: ${{ needs.build-settings.outputs.ref }} + stream: ${{ inputs.stream }} + version: ${{ needs.build-settings.outputs.imageVersion }} + add_latest: true diff --git a/.github/workflows/on-release.yml b/.github/workflows/on-release.yml index a77b080c9..b3e3bee73 100644 --- a/.github/workflows/on-release.yml +++ b/.github/workflows/on-release.yml @@ -37,12 +37,16 @@ jobs: role-to-assume: arn:aws:iam::795746500882:role/GithubAddReleaseVersion aws-region: eu-central-1 - - name: Update OS images - working-directory: hack/add-version + - name: Build versionsapi CLI + working-directory: internal/versionsapi/cli + run: go build -o versionsapi + + - name: Add version to versionsapi + working-directory: internal/versionsapi/cli run: | latest=$([[ "${{ inputs.latest }}" = "true" ]] && echo "--latest" || echo "") - go run main.go \ - --version "${{ github.event.release.tag_name }}${{ github.event.inputs.tag }}" \ - --stream stable \ + ./versionsapi add \ --release \ + --stream "stable" \ + --version "${{ github.event.release.tag_name }}${{ github.event.inputs.tag }}" \ "${latest}" diff --git a/hack/add-version/main.go b/hack/add-version/main.go deleted file mode 100644 index d8ebf9b13..000000000 --- a/hack/add-version/main.go +++ /dev/null @@ -1,485 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -// add-version adds a new constellation release version to the list of available versions. -// It is meant to be run by the CI pipeline to make new versions available / discoverable. -package main - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "path" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - awsconfig "github.com/aws/aws-sdk-go-v2/config" - s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" - "github.com/aws/aws-sdk-go-v2/service/cloudfront" - cftypes "github.com/aws/aws-sdk-go-v2/service/cloudfront/types" - "github.com/aws/aws-sdk-go-v2/service/s3" - s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/logger" - "github.com/edgelesssys/constellation/v2/internal/versionsapi-old" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "golang.org/x/mod/semver" -) - -var errVersionListMissing = errors.New("version list does not exist") - -const ( - skipRefStr = "-" - imageKind = "image" - defaultRegion = "eu-central-1" - defaultBucket = "cdn-constellation-backend" - defaultDistributionID = "E1H77EZTHC3NE4" - maxCacheInvalidationWaitTime = 5 * time.Minute -) - -func main() { - log := logger.New(logger.JSONLog, zapcore.InfoLevel) - ctx := context.Background() - - flags := flags{ - version: flag.String("version", "", "Version to add (format: \"v1.2.3\")"), - stream: flag.String("stream", "", "Stream to add the version to"), - ref: flag.String("ref", "", "Ref to add the version to"), - release: flag.Bool("release", false, "Whether the version is a release"), - latest: flag.Bool("latest", false, "Whether to set this version as the new latest version"), - dryRun: flag.Bool("dryrun", false, "Whether to run in dry-run mode (no changes are made)"), - region: flag.String("region", defaultRegion, "AWS region"), - bucket: flag.String("bucket", defaultBucket, "S3 bucket"), - distributionID: flag.String("distribution-id", defaultDistributionID, "cloudfront distribution id"), - } - flag.Parse() - if err := flags.validate(); err != nil { - log.With(zap.Error(err)).Fatalf("Invalid flags") - } - - updateFetcher := versionsapi.New() - versionManager, err := newVersionManager(ctx, *flags.region, *flags.bucket, *flags.distributionID, *flags.dryRun, log) - if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to create version uploader") - } - - ver := version{ - versionStr: *flags.version, - stream: *flags.stream, - ref: *flags.ref, - } - - if err := ensureMinorVersion(ctx, versionManager, ver, log); err != nil { - log.With(zap.Error(err)).Fatalf("Failed to ensure minor version") - } - - added, err := ensurePatchVersion(ctx, versionManager, ver, log) - if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to ensure patch version") - } - - if added && *flags.latest { - if err := versionManager.addLatest(ctx, ver); err != nil { - log.With(zap.Error(err)).Fatalf("Failed to update latest version object") - } - log.Infof("Added %q as latest version.", ver) - } - - log.Infof("Major to minor url: %s", ver.URL(granularityMajor)) - log.Infof("Minor to patch url: %s", ver.URL(granularityMinor)) - - if !versionManager.dirty { - log.Infof("No changes made, everything up to date.") - return - } - log.Infof("Successfully added version %q", *flags.version) - - log.Infof("Waiting for cache invalidation.") - if err := versionManager.invalidateCaches(ctx, ver, *flags.latest); err != nil { - log.With(zap.Error(err)).Fatalf("Failed to invalidate caches") - } - - waitForCacheUpdate(ctx, updateFetcher, ver, log) -} - -func ensureMinorVersion(ctx context.Context, versionManager *versionManager, ver version, log *logger.Logger) error { - minorVerList, err := versionManager.getVersionList(ctx, ver, granularityMajor) - log.Debugf("Minor version list: %v", minorVerList) - if errors.Is(err, errVersionListMissing) { - log.Infof("Version list for minor versions under %q does not exist. Creating new list.", ver.Major()) - minorVerList = &versionsapi.List{ - Ref: ver.Ref(), - Stream: ver.Stream(), - Granularity: "major", - Base: ver.Major(), - Kind: imageKind, - Versions: []string{}, - } - } else if err != nil { - return fmt.Errorf("failed to list minor versions: %w", err) - } - - if minorVerList.Contains(ver.MajorMinor()) { - log.Infof("Version %q already exists in list %v.", ver.MajorMinor(), minorVerList.Versions) - return nil - } - - minorVerList.Versions = append(minorVerList.Versions, ver.MajorMinor()) - log.Debugf("New minor version list: %v", minorVerList) - - if err := versionManager.updateVersionList(ctx, minorVerList); err != nil { - return fmt.Errorf("failed to add minor version: %w", err) - } - - log.Infof("Added %q to list.", ver.MajorMinor()) - return nil -} - -func ensurePatchVersion(ctx context.Context, versionManager *versionManager, ver version, log *logger.Logger) (bool, error) { - pathVerList, err := versionManager.getVersionList(ctx, ver, granularityMinor) - if errors.Is(err, errVersionListMissing) { - log.Infof("Version list for patch versions under %q does not exist. Creating new list.", ver.MajorMinor()) - pathVerList = &versionsapi.List{ - Ref: ver.Ref(), - Stream: ver.Stream(), - Granularity: "minor", - Base: ver.MajorMinor(), - Kind: imageKind, - Versions: []string{}, - } - } else if err != nil { - return false, fmt.Errorf("failed to get patch versions: %w", err) - } - - if pathVerList.Contains(ver.String()) { - log.Infof("Version %q already exists in list %v.", ver.String(), pathVerList.Versions) - return false, nil - } - - pathVerList.Versions = append(pathVerList.Versions, ver.String()) - - if err := versionManager.updateVersionList(ctx, pathVerList); err != nil { - log.With(zap.Error(err)).Fatalf("Failed to add patch version") - } - - log.Infof("Added %q to list.", ver.String()) - return true, nil -} - -type version struct { - versionStr string - stream string - ref string -} - -func (v *version) String() string { - return semver.Canonical(v.versionStr) -} - -func (v *version) Major() string { - return semver.Major(v.versionStr) -} - -func (v *version) MajorMinor() string { - return semver.MajorMinor(v.versionStr) -} - -func (v *version) WithGranularity(gran granularity) string { - switch gran { - case granularityMajor: - return v.Major() - case granularityMinor: - return v.MajorMinor() - default: - return "" - } -} - -func (v *version) URL(gran granularity) string { - return constants.CDNRepositoryURL + "/" + v.JSONPath(gran) -} - -func (v *version) JSONPath(gran granularity) string { - return path.Join(constants.CDNAPIPrefix, "ref", v.ref, "stream", v.stream, "versions", gran.String(), v.WithGranularity(gran), imageKind+".json") -} - -func (v *version) Stream() string { - return v.stream -} - -func (v *version) Ref() string { - return v.ref -} - -type flags struct { - version *string - stream *string - ref *string - release *bool - latest *bool - dryRun *bool - region *string - bucket *string - distributionID *string -} - -func (f *flags) validate() error { - if err := validateVersion(*f.version); err != nil { - return err - } - - if *f.ref == "" && !*f.release { - if !*f.release { - return fmt.Errorf("branch flag must be set for non-release versions") - } - } - - if *f.ref != "" && *f.release { - return fmt.Errorf("branch flag must not be set for release versions") - } - - if *f.release { - *f.ref = skipRefStr - } else { - *f.latest = true // always set latest for non-release versions - } - - ref := versionsapi.CanonicalRef(*f.ref) - if !versionsapi.IsValidRef(ref) { - return fmt.Errorf("invalid ref %q", *f.ref) - } - *f.ref = ref - - if !versionsapi.IsValidStream(*f.ref, *f.stream) { - return fmt.Errorf("invalid stream %q for ref %q", *f.stream, *f.ref) - } - - return nil -} - -func validateVersion(version string) error { - if !semver.IsValid(version) { - return fmt.Errorf("version %q is not a valid semantic version", version) - } - if semver.Canonical(version) != version { - return fmt.Errorf("version %q is not a canonical semantic version", version) - } - return nil -} - -func ensureMinorVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher, ver version) error { - existingMinorVersions, err := fetcher.MinorVersionsOf(ctx, ver.Ref(), ver.Stream(), ver.Major(), imageKind) - if err != nil { - return err - } - if !existingMinorVersions.Contains(ver.MajorMinor()) { - return errors.New("minor version does not exist") - } - return nil -} - -func ensurePatchVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher, ver version) error { - existingPatchVersions, err := fetcher.PatchVersionsOf(ctx, ver.Ref(), ver.Stream(), ver.MajorMinor(), imageKind) - if err != nil { - return err - } - if !existingPatchVersions.Contains(ver.String()) { - return errors.New("patch version does not exist") - } - return nil -} - -type versionManager struct { - config aws.Config - cloudfrontc *cloudfront.Client - s3c *s3.Client - uploader *s3manager.Uploader - bucket string - distributionID string - dirty bool // manager gets dirty on write - dryRun bool // no write operations - log *logger.Logger -} - -func newVersionManager(ctx context.Context, region, bucket, distributionID string, dryRun bool, log *logger.Logger) (*versionManager, error) { - cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) - if err != nil { - return nil, err - } - cloudfrontc := cloudfront.NewFromConfig(cfg) - s3c := s3.NewFromConfig(cfg) - uploader := s3manager.NewUploader(s3c) - return &versionManager{ - config: cfg, - cloudfrontc: cloudfrontc, - s3c: s3c, - uploader: uploader, - bucket: bucket, - distributionID: distributionID, - dryRun: dryRun, - log: log, - }, nil -} - -func (m *versionManager) getVersionList(ctx context.Context, ver version, gran granularity) (*versionsapi.List, error) { - in := &s3.GetObjectInput{ - Bucket: aws.String(m.bucket), - Key: aws.String(ver.JSONPath(gran)), - } - out, err := m.s3c.GetObject(ctx, in) - var noSuchkey *s3types.NoSuchKey - if errors.As(err, &noSuchkey) { - return nil, errVersionListMissing - } else if err != nil { - return nil, err - } - defer out.Body.Close() - - var list versionsapi.List - if err := json.NewDecoder(out.Body).Decode(&list); err != nil { - return nil, err - } - - return &list, nil -} - -func (m *versionManager) updateVersionList(ctx context.Context, list *versionsapi.List) error { - semver.Sort(list.Versions) - if err := list.Validate(); err != nil { - return err - } - - rawList, err := json.Marshal(list) - if err != nil { - return err - } - - in := &s3.PutObjectInput{ - Bucket: aws.String(m.bucket), - Key: aws.String(list.JSONPath()), - Body: bytes.NewBuffer(rawList), - } - - if m.dryRun { - m.log.Infof("dryRun: s3 put object {Bucket: %v, Key: %v, Body: %v", m.bucket, list.JSONPath(), string(rawList)) - return nil - } - - m.dirty = true - - _, err = m.uploader.Upload(ctx, in) - - return err -} - -func (m *versionManager) addLatest(ctx context.Context, ver version) error { - latest := &versionsapi.Latest{ - Ref: ver.Ref(), - Stream: ver.Stream(), - Kind: imageKind, - Version: ver.String(), - } - if err := latest.Validate(); err != nil { - return err - } - - rawLatest, err := json.Marshal(latest) - if err != nil { - return err - } - - in := &s3.PutObjectInput{ - Bucket: aws.String(m.bucket), - Key: aws.String(latest.JSONPath()), - Body: bytes.NewBuffer(rawLatest), - } - - if m.dryRun { - m.log.Infof("dryRun: s3 put object {Bucket: %v, Key: %v, Body: %v", m.bucket, latest.JSONPath(), string(rawLatest)) - return nil - } - - m.dirty = true - - _, err = m.uploader.Upload(ctx, in) - - return err -} - -func (m *versionManager) invalidateCaches(ctx context.Context, ver version, latest bool) error { - invalidIn := &cloudfront.CreateInvalidationInput{ - DistributionId: aws.String(m.distributionID), - InvalidationBatch: &cftypes.InvalidationBatch{ - CallerReference: aws.String(fmt.Sprintf("%d", time.Now().Unix())), - Paths: &cftypes.Paths{ - Quantity: aws.Int32(2), - Items: []string{ - "/" + ver.URL(granularityMajor), - "/" + ver.URL(granularityMinor), - }, - }, - }, - } - if latest { - invalidIn.InvalidationBatch.Paths.Quantity = aws.Int32(3) - path := path.Join("ref", ver.Ref(), "stream", ver.Stream(), "versions/latest/image.json") - invalidIn.InvalidationBatch.Paths.Items = append(invalidIn.InvalidationBatch.Paths.Items, "/"+path) - } - invalidation, err := m.cloudfrontc.CreateInvalidation(ctx, invalidIn) - if err != nil { - return err - } - - waiter := cloudfront.NewInvalidationCompletedWaiter(m.cloudfrontc) - waitIn := &cloudfront.GetInvalidationInput{ - DistributionId: aws.String(m.distributionID), - Id: invalidation.Invalidation.Id, - } - if err := waiter.Wait(ctx, waitIn, maxCacheInvalidationWaitTime); err != nil { - return err - } - - return nil -} - -func waitForCacheUpdate(ctx context.Context, updateFetcher *versionsapi.Fetcher, ver version, log *logger.Logger) { - sawAddedVersions := true - if err := ensureMinorVersionExists(ctx, updateFetcher, ver); err != nil { - sawAddedVersions = false - log.Warnf("Failed to ensure minor version exists: %v. This may be resolved by waiting.", err) - } - - if err := ensurePatchVersionExists(ctx, updateFetcher, ver); err != nil { - sawAddedVersions = false - log.Warnf("Failed to ensure patch version exists: %v. This may be resolved by waiting.", err) - } - - if sawAddedVersions { - log.Infof("Versions are available via API.") - } -} - -type granularity int - -const ( - granularityMajor granularity = iota - granularityMinor -) - -func (g granularity) String() string { - switch g { - case granularityMajor: - return "major" - case granularityMinor: - return "minor" - default: - return "unknown" - } -} diff --git a/hack/go.mod b/hack/go.mod index 29e79b76c..2bf4fb963 100644 --- a/hack/go.mod +++ b/hack/go.mod @@ -79,24 +79,22 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/aws/aws-sdk-go-v2 v1.17.3 + github.com/aws/aws-sdk-go-v2 v1.17.3 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/config v1.18.7 + github.com/aws/aws-sdk-go-v2/config v1.18.7 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.7 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 // indirect - github.com/aws/aws-sdk-go-v2/service/cloudfront v1.22.2 github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21 // indirect github.com/aws/aws-sdk-go-v2/service/kms v1.19.4 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6 + github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 // indirect diff --git a/hack/go.sum b/hack/go.sum index be1028be3..720d73eb3 100644 --- a/hack/go.sum +++ b/hack/go.sum @@ -209,8 +209,6 @@ github.com/aws/aws-sdk-go-v2/credentials v1.13.7 h1:qUUcNS5Z1092XBFT66IJM7mYkMwg github.com/aws/aws-sdk-go-v2/credentials v1.13.7/go.mod h1:AdCcbZXHQCjJh6NaH3pFaw8LUeBFn5+88BZGMVGuBT8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46 h1:OCX1pQ4pcqhsDV7B92HzdLWjHWOQsILvjLinpaUWhcc= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46/go.mod h1:MxCBOcyNXGJRvfpPiH+L6n/BF9zbowthGSUZdDvQF/c= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE= @@ -219,8 +217,6 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 h1:H/mF2LNWwX00lD6FlYfKpLLZgUW7oIzCBkig78x4Xok= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18/go.mod h1:T2Ku+STrYQ1zIkL1wMvj8P3wWQaaCMKNdz70MT2FLfE= -github.com/aws/aws-sdk-go-v2/service/cloudfront v1.22.2 h1:XP88fE1Y8a5308lYzmTnC0be6prtaKAXc6qlD1e4YIE= -github.com/aws/aws-sdk-go-v2/service/cloudfront v1.22.2/go.mod h1:xUOmvPrMKmH94stXswKsGSkL02vMpNU+rTG+eIzFfNQ= github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 h1:m6HYlpZlTWb9vHuuRHpWRieqPHWlS0mvQ90OJNrG/Nk= github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0/go.mod h1:mV0E7631M1eXdB+tlGFIw6JxfsC7Pz7+7Aw15oLVhZw= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= diff --git a/internal/versionsapi-old/versionsapi.go b/internal/versionsapi-old/versionsapi.go deleted file mode 100644 index 511078aa6..000000000 --- a/internal/versionsapi-old/versionsapi.go +++ /dev/null @@ -1,278 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package versionsapi - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "path" - "regexp" - "strings" - - "github.com/edgelesssys/constellation/v2/internal/constants" - "golang.org/x/mod/semver" -) - -// List represents a list of versions for a kind of resource. -// It has a granularity of either "major" or "minor". -// -// For example, a List with granularity "major" could contain -// the base version "v1" and a list of minor versions "v1.0", "v1.1", "v1.2" etc. -// A List with granularity "minor" could contain the base version -// "v1.0" and a list of patch versions "v1.0.0", "v1.0.1", "v1.0.2" etc. -type List struct { - // Ref is the branch name the list belongs to. - Ref string `json:"ref,omitempty"` - // Stream is the update stream of the list. - Stream string `json:"stream,omitempty"` - // Granularity is the granularity of the base version of this list. - // It can be either "major" or "minor". - Granularity string `json:"granularity,omitempty"` - // Base is the base version of the list. - // Every version in the list is a finer-grained version of this base version. - Base string `json:"base,omitempty"` - // Kind is the kind of resource this list is for. - Kind string `json:"kind,omitempty"` - // Versions is a list of all versions in this list. - Versions []string `json:"versions,omitempty"` -} - -// Validate checks if the list is valid. -// This performs the following checks: -// - The ref is set. -// - The stream is supported. -// - The granularity is "major" or "minor". -// - The kind is supported. -// - The base version is a valid semantic version that matches the granularity. -// - All versions in the list are valid semantic versions that are finer-grained than the base version. -func (l *List) Validate() error { - var issues []string - if !IsValidRef(l.Ref) { - issues = append(issues, "ref is empty") - } - if !IsValidStream(l.Ref, l.Stream) { - issues = append(issues, fmt.Sprintf("stream %q is not supported on ref %q", l.Stream, l.Ref)) - } - if l.Granularity != "major" && l.Granularity != "minor" { - issues = append(issues, fmt.Sprintf("granularity %q is not supported", l.Granularity)) - } - if l.Kind != "image" { - issues = append(issues, fmt.Sprintf("kind %q is not supported", l.Kind)) - } - if !semver.IsValid(l.Base) { - issues = append(issues, fmt.Sprintf("base version %q is not a valid semantic version", l.Base)) - } - var normalizeFunc func(string) string - switch l.Granularity { - case "major": - normalizeFunc = semver.Major - case "minor": - normalizeFunc = semver.MajorMinor - default: - normalizeFunc = func(s string) string { return s } - } - if normalizeFunc(l.Base) != l.Base { - issues = append(issues, fmt.Sprintf("base version %q is not a %v version", l.Base, l.Granularity)) - } - for _, ver := range l.Versions { - if !semver.IsValid(ver) { - issues = append(issues, fmt.Sprintf("version %q in list is not a valid semantic version", ver)) - } - if normalizeFunc(ver) != l.Base { - issues = append(issues, fmt.Sprintf("version %q in list is not a finer-grained version of base version %q", ver, l.Base)) - } - } - if len(issues) > 0 { - return fmt.Errorf("version list is invalid:\n%s", strings.Join(issues, "\n")) - } - return nil -} - -// JSONPath returns the S3 JSON path for this object. -func (l *List) JSONPath() string { - return path.Join(constants.CDNAPIPrefix, "ref", l.Ref, "stream", l.Stream, "versions", l.Granularity, l.Base, l.Kind+".json") -} - -// Latest is the latest version of a kind of resource. -type Latest struct { - // Ref is the branch name this latest version belongs to. - Ref string `json:"ref,omitempty"` - // Stream is stream name this latest version belongs to. - Stream string `json:"stream,omitempty"` - // Kind is the kind of resource this latest version is for. - Kind string `json:"kind,omitempty"` - // Version is the latest version for this ref, stream and kind. - Version string `json:"version,omitempty"` -} - -// Validate checks if this latest version is valid. -func (l *Latest) Validate() error { - var issues []string - if !IsValidRef(l.Ref) { - issues = append(issues, "ref is empty") - } - if !IsValidStream(l.Ref, l.Stream) { - issues = append(issues, fmt.Sprintf("stream %q is not supported on ref %q", l.Stream, l.Ref)) - } - if l.Kind != "image" { - issues = append(issues, fmt.Sprintf("kind %q is not supported", l.Kind)) - } - if !semver.IsValid(l.Version) { - issues = append(issues, fmt.Sprintf("version %q is not a valid semantic version", l.Version)) - } - if len(issues) > 0 { - return fmt.Errorf("latest version is invalid:\n%s", strings.Join(issues, "\n")) - } - return nil -} - -// JSONPath returns the S3 JSON path for this object. -func (l *Latest) JSONPath() string { - return path.Join(constants.CDNAPIPrefix, "ref", l.Ref, "stream", l.Stream, "versions", "latest", l.Kind+".json") -} - -// Contains returns true if the list contains the given version. -func (l *List) Contains(version string) bool { - for _, v := range l.Versions { - if v == version { - return true - } - } - return false -} - -// Fetcher fetches a list of versions. -type Fetcher struct { - httpc httpc -} - -// New returns a new VersionsFetcher. -func New() *Fetcher { - return &Fetcher{ - httpc: http.DefaultClient, - } -} - -// MinorVersionsOf fetches the list of minor versions for a given stream, major version and kind. -func (f *Fetcher) MinorVersionsOf(ctx context.Context, ref, stream, major, kind string) (*List, error) { - return f.list(ctx, ref, stream, "major", major, kind) -} - -// PatchVersionsOf fetches the list of patch versions for a given stream, minor version and kind. -func (f *Fetcher) PatchVersionsOf(ctx context.Context, ref, stream, minor, kind string) (*List, error) { - return f.list(ctx, ref, stream, "minor", minor, kind) -} - -// list fetches the list of versions for a given stream, granularity, base and kind. -func (f *Fetcher) list(ctx context.Context, ref, stream, granularity, base, kind string) (*List, error) { - raw, err := getFromURL(ctx, f.httpc, ref, stream, granularity, base, kind) - if err != nil { - return nil, fmt.Errorf("fetching versions list: %w", err) - } - list := &List{} - if err := json.Unmarshal(raw, &list); err != nil { - return nil, fmt.Errorf("decoding versions list: %w", err) - } - if err := list.Validate(); err != nil { - return nil, fmt.Errorf("validating versions list: %w", err) - } - if !f.listMatchesRequest(list, stream, granularity, base, kind) { - return nil, fmt.Errorf("versions list does not match request") - } - return list, nil -} - -func (f *Fetcher) listMatchesRequest(list *List, stream, granularity, base, kind string) bool { - return list.Stream == stream && list.Granularity == granularity && list.Base == base && list.Kind == kind -} - -// getFromURL fetches the versions list from a URL. -func getFromURL(ctx context.Context, client httpc, ref, stream, granularity, base, kind string) ([]byte, error) { - url, err := url.Parse(constants.CDNRepositoryURL) - if err != nil { - return nil, fmt.Errorf("parsing image version repository URL: %w", err) - } - kindFilename := path.Base(kind) + ".json" - url.Path = path.Join(constants.CDNAPIPrefix, "ref", ref, "stream", stream, "versions", granularity, base, kindFilename) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), http.NoBody) - if err != nil { - return nil, err - } - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - switch resp.StatusCode { - case http.StatusNotFound: - return nil, fmt.Errorf("versions list %q does not exist", url.String()) - default: - return nil, fmt.Errorf("unexpected status code %d", resp.StatusCode) - } - } - content, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - return content, nil -} - -// IsValidStream returns true if the given stream is a valid stream. -func IsValidStream(ref, stream string) bool { - validReleaseStreams := []string{"stable", "console", "debug"} - validStreams := []string{"nightly", "console", "debug"} - - if isReleaseRef(ref) { - validStreams = validReleaseStreams - } - - for _, validStream := range validStreams { - if stream == validStream { - return true - } - } - - return false -} - -var notAZ09Regexp = regexp.MustCompile("[^a-zA-Z0-9-]+") - -// CanonicalRef returns the canonicalized ref for the given ref. -func CanonicalRef(ref string) string { - return notAZ09Regexp.ReplaceAllString(ref, "-") -} - -// IsValidRef returns true if the given ref is a valid ref. -func IsValidRef(ref string) bool { - if ref == "" { - return false - } - - if notAZ09Regexp.FindString(ref) != "" { - return false - } - - if strings.HasPrefix(ref, "refs-heads") { - return false - } - - return true -} - -func isReleaseRef(ref string) bool { - return ref == "-" -} - -type httpc interface { - Do(req *http.Request) (*http.Response, error) -} diff --git a/internal/versionsapi-old/versionsapi_test.go b/internal/versionsapi-old/versionsapi_test.go deleted file mode 100644 index 7c6a6f6ff..000000000 --- a/internal/versionsapi-old/versionsapi_test.go +++ /dev/null @@ -1,347 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package versionsapi - -import ( - "bytes" - "context" - "encoding/json" - "io" - "net/http" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/goleak" -) - -func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) -} - -func TestValidate(t *testing.T) { - testCases := map[string]struct { - listFunc func() *List - overrideFunc func(list *List) - wantErr bool - }{ - "valid major list": { - listFunc: majorList, - }, - "valid minor list": { - listFunc: minorList, - }, - "invalid stream": { - listFunc: majorList, - overrideFunc: func(list *List) { - list.Stream = "invalid" - }, - wantErr: true, - }, - "invalid granularity": { - listFunc: majorList, - overrideFunc: func(list *List) { - list.Granularity = "invalid" - }, - wantErr: true, - }, - "invalid kind": { - listFunc: majorList, - overrideFunc: func(list *List) { - list.Kind = "invalid" - }, - wantErr: true, - }, - "base ver is not semantic version": { - listFunc: majorList, - overrideFunc: func(list *List) { - list.Base = "invalid" - }, - wantErr: true, - }, - "base ver does not reflect major granularity": { - listFunc: majorList, - overrideFunc: func(list *List) { - list.Base = "v1.0" - }, - wantErr: true, - }, - "base ver does not reflect minor granularity": { - listFunc: minorList, - overrideFunc: func(list *List) { - list.Base = "v1" - }, - wantErr: true, - }, - "version in list is not semantic version": { - listFunc: majorList, - overrideFunc: func(list *List) { - list.Versions[0] = "invalid" - }, - wantErr: true, - }, - "version in list is not sub version of base": { - listFunc: majorList, - overrideFunc: func(list *List) { - list.Versions[0] = "v2.1" - }, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - list := tc.listFunc() - if tc.overrideFunc != nil { - tc.overrideFunc(list) - } - err := list.Validate() - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - }) - } -} - -func TestList(t *testing.T) { - majorListJSON, err := json.Marshal(majorList()) - require.NoError(t, err) - minorListJSON, err := json.Marshal(minorList()) - require.NoError(t, err) - inconsistentList := majorList() - inconsistentList.Base = "v2" - inconsistentListJSON, err := json.Marshal(inconsistentList) - require.NoError(t, err) - client := newTestClient(func(req *http.Request) *http.Response { - switch req.URL.Path { - case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/image.json": - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBuffer(majorListJSON)), - Header: make(http.Header), - } - case "/constellation/v1/ref/test-ref/stream/nightly/versions/minor/v1.1/image.json": - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBuffer(minorListJSON)), - Header: make(http.Header), - } - case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/500.json": // 500 error - return &http.Response{ - StatusCode: http.StatusInternalServerError, - Body: io.NopCloser(bytes.NewBufferString("Server Error.")), - Header: make(http.Header), - } - case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/nojson.json": // invalid format - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBufferString("not json")), - Header: make(http.Header), - } - case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v2/image.json": // inconsistent list - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBuffer(inconsistentListJSON)), - Header: make(http.Header), - } - case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v3/image.json": // does not match requested version - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBuffer(minorListJSON)), - Header: make(http.Header), - } - } - return &http.Response{ - StatusCode: http.StatusNotFound, - Body: io.NopCloser(bytes.NewBufferString("Not found.")), - Header: make(http.Header), - } - }) - - testCases := map[string]struct { - ref, stream, granularity, base, kind string - overrideFile string - wantList List - wantErr bool - }{ - "major list fetched remotely": { - wantList: *majorList(), - }, - "minor list fetched remotely": { - granularity: "minor", - base: "v1.1", - wantList: *minorList(), - }, - "list does not exist": { - stream: "unknown", - wantErr: true, - }, - "unexpected error code": { - kind: "500", - wantErr: true, - }, - "invalid json returned": { - kind: "nojson", - wantErr: true, - }, - "invalid list returned": { - base: "v2", - wantErr: true, - }, - "response does not match request": { - base: "v3", - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - ref := "test-ref" - stream := "nightly" - granularity := "major" - base := "v1" - kind := "image" - if tc.stream != "" { - stream = tc.stream - } - if tc.granularity != "" { - granularity = tc.granularity - } - if tc.base != "" { - base = tc.base - } - if tc.kind != "" { - kind = tc.kind - } - - fetcher := &Fetcher{ - httpc: client, - } - list, err := fetcher.list(context.Background(), ref, stream, granularity, base, kind) - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantList, *list) - }) - } -} - -// roundTripFunc . -type roundTripFunc func(req *http.Request) *http.Response - -// RoundTrip . -func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { - return f(req), nil -} - -// newTestClient returns *http.Client with Transport replaced to avoid making real calls. -func newTestClient(fn roundTripFunc) *http.Client { - return &http.Client{ - Transport: fn, - } -} - -func majorList() *List { - return &List{ - Ref: "test-ref", - Stream: "nightly", - Granularity: "major", - Base: "v1", - Kind: "image", - Versions: []string{ - "v1.0", "v1.1", "v1.2", - }, - } -} - -func minorList() *List { - return &List{ - Ref: "test-ref", - Stream: "nightly", - Granularity: "minor", - Base: "v1.1", - Kind: "image", - Versions: []string{ - "v1.1.0", "v1.1.1", "v1.1.2", - }, - } -} - -func TestIsValidRef(t *testing.T) { - testCases := map[string]bool{ - "feat/foo": false, - "feat-foo": true, - "feat$foo": false, - "3234": true, - "feat foo": false, - "refs-heads-feat-foo": false, - "": false, - } - - for ref, want := range testCases { - t.Run(ref, func(t *testing.T) { - assert := assert.New(t) - assert.Equal(want, IsValidRef(ref)) - }) - } -} - -func TestCanonicalRef(t *testing.T) { - testCases := map[string]string{ - "feat/foo": "feat-foo", - "feat-foo": "feat-foo", - "feat$foo": "feat-foo", - "3234": "3234", - "feat foo": "feat-foo", - } - - for ref, want := range testCases { - t.Run(ref, func(t *testing.T) { - assert := assert.New(t) - assert.Equal(want, CanonicalRef(ref)) - }) - } -} - -func TestIsValidStream(t *testing.T) { - testCases := []struct { - branch string - stream string - want bool - }{ - {branch: "-", stream: "stable", want: true}, - {branch: "-", stream: "debug", want: true}, - {branch: "-", stream: "nightly", want: false}, - {branch: "-", stream: "console", want: true}, - {branch: "main", stream: "stable", want: false}, - {branch: "main", stream: "debug", want: true}, - {branch: "main", stream: "nightly", want: true}, - {branch: "main", stream: "console", want: true}, - {branch: "foo-branch", stream: "nightly", want: true}, - {branch: "foo-branch", stream: "console", want: true}, - {branch: "foo-branch", stream: "debug", want: true}, - {branch: "foo-branch", stream: "stable", want: false}, - } - - for _, tc := range testCases { - t.Run(tc.branch+"+"+tc.stream, func(t *testing.T) { - assert := assert.New(t) - - assert.Equal(tc.want, IsValidStream(tc.branch, tc.stream)) - }) - } -}