diff --git a/cli/internal/cmd/upgradeplan.go b/cli/internal/cmd/upgradeplan.go index a34a092b1..8e4fbc674 100644 --- a/cli/internal/cmd/upgradeplan.go +++ b/cli/internal/cmd/upgradeplan.go @@ -110,7 +110,7 @@ func upgradePlan(cmd *cobra.Command, planner upgradePlanner, patchLister patchLi var updateCandidates []string for _, minorVer := range allowedMinorVersions { - versionList, err := patchLister.PatchVersionsOf(cmd.Context(), "stable", minorVer, "image") + versionList, err := patchLister.PatchVersionsOf(cmd.Context(), "-", "stable", minorVer, "image") if err == nil { updateCandidates = append(updateCandidates, versionList.Versions...) } @@ -335,5 +335,5 @@ type upgradePlanner interface { } type patchLister interface { - PatchVersionsOf(ctx context.Context, stream, minor, kind string) (*versionsapi.List, error) + PatchVersionsOf(ctx context.Context, ref, stream, minor, kind string) (*versionsapi.List, error) } diff --git a/cli/internal/cmd/upgradeplan_test.go b/cli/internal/cmd/upgradeplan_test.go index f2fbf6acd..df8e845e5 100644 --- a/cli/internal/cmd/upgradeplan_test.go +++ b/cli/internal/cmd/upgradeplan_test.go @@ -492,6 +492,6 @@ type stubPatchLister struct { err error } -func (s stubPatchLister) PatchVersionsOf(ctx context.Context, stream, minor, kind string) (*versionsapi.List, error) { +func (s stubPatchLister) PatchVersionsOf(ctx context.Context, ref, stream, minor, kind string) (*versionsapi.List, error) { return &s.list, s.err } diff --git a/hack/add-version/main.go b/hack/add-version/main.go index 37ae05cfd..97c27337d 100644 --- a/hack/add-version/main.go +++ b/hack/add-version/main.go @@ -36,6 +36,7 @@ import ( var errVersionListMissing = errors.New("version list does not exist") const ( + skipRefStr = "-" imageKind = "image" defaultRegion = "eu-central-1" defaultBucket = "cdn-constellation-backend" @@ -50,6 +51,9 @@ func main() { flags := flags{ version: flag.String("version", "", "Version to add (format: \"v1.2.3\")"), stream: flag.String("stream", "", "Stream to add the version to"), + ref: flag.String("ref", "", "Ref to add the version to"), + release: flag.Bool("release", false, "Whether the version is a release"), + dryRun: flag.Bool("dryrun", false, "Whether to run in dry-run mode (no changes are made)"), region: flag.String("region", defaultRegion, "AWS region"), bucket: flag.String("bucket", defaultBucket, "S3 bucket"), distributionID: flag.String("distribution-id", defaultDistributionID, "cloudfront distribution id"), @@ -60,7 +64,7 @@ func main() { } updateFetcher := versionsapi.New() - versionManager, err := newVersionManager(ctx, *flags.region, *flags.bucket, *flags.distributionID) + versionManager, err := newVersionManager(ctx, *flags.region, *flags.bucket, *flags.distributionID, *flags.dryRun, log) if err != nil { log.With(zap.Error(err)).Fatalf("Failed to create version uploader") } @@ -68,6 +72,7 @@ func main() { ver := version{ versionStr: *flags.version, stream: *flags.stream, + ref: *flags.ref, } if err := ensureMinorVersion(ctx, versionManager, ver, log); err != nil { @@ -101,6 +106,7 @@ func ensureMinorVersion(ctx context.Context, versionManager *versionManager, ver if errors.Is(err, errVersionListMissing) { log.Infof("Version list for minor versions under %q does not exist. Creating new list.", ver.Major()) minorVerList = &versionsapi.List{ + Ref: ver.Ref(), Stream: ver.Stream(), Granularity: "major", Base: ver.Major(), @@ -132,6 +138,7 @@ func ensurePatchVersion(ctx context.Context, versionManager *versionManager, ver if errors.Is(err, errVersionListMissing) { log.Infof("Version list for patch versions under %q does not exist. Creating new list.", ver.MajorMinor()) pathVerList = &versionsapi.List{ + Ref: ver.Ref(), Stream: ver.Stream(), Granularity: "minor", Base: ver.MajorMinor(), @@ -160,6 +167,7 @@ func ensurePatchVersion(ctx context.Context, versionManager *versionManager, ver type version struct { versionStr string stream string + ref string } func (v *version) MajorMinorPatch() string { @@ -190,16 +198,23 @@ func (v *version) URL(gran granularity) string { } func (v *version) JSONPath(gran granularity) string { - return path.Join(constants.CDNVersionsPath, "stream", v.stream, gran.String(), v.WithGranularity(gran), imageKind+".json") + return path.Join(constants.CDNAPIPrefix, "ref", v.ref, "stream", v.stream, "versions", gran.String(), v.WithGranularity(gran), imageKind+".json") } func (v *version) Stream() string { return v.stream } +func (v *version) Ref() string { + return v.ref +} + type flags struct { version *string stream *string + ref *string + release *bool + dryRun *bool region *string bucket *string distributionID *string @@ -209,9 +224,29 @@ func (f *flags) validate() error { if err := validateVersion(*f.version); err != nil { return err } - if *f.stream == "" { - return errors.New("stream must be set") + + if *f.ref == "" && !*f.release { + if !*f.release { + return fmt.Errorf("branch flag must be set for non-release versions") + } } + if *f.ref != "" && *f.release { + return fmt.Errorf("branch flag must not be set for release versions") + } + if *f.release { + *f.ref = skipRefStr + } + + ref := versionsapi.CanonicalRef(*f.ref) + if !versionsapi.IsValidRef(ref) { + return fmt.Errorf("invalid ref %q", *f.ref) + } + *f.ref = ref + + if !versionsapi.IsValidStream(*f.ref, *f.stream) { + return fmt.Errorf("invalid stream %q for ref %q", *f.stream, *f.ref) + } + return nil } @@ -226,7 +261,7 @@ func validateVersion(version string) error { } func ensureMinorVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher, ver version) error { - existingMinorVersions, err := fetcher.MinorVersionsOf(ctx, ver.Stream(), ver.Major(), imageKind) + existingMinorVersions, err := fetcher.MinorVersionsOf(ctx, ver.Ref(), ver.Stream(), ver.Major(), imageKind) if err != nil { return err } @@ -237,7 +272,7 @@ func ensureMinorVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher, } func ensurePatchVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher, ver version) error { - existingPatchVersions, err := fetcher.PatchVersionsOf(ctx, ver.Stream(), ver.MajorMinor(), imageKind) + existingPatchVersions, err := fetcher.PatchVersionsOf(ctx, ver.Ref(), ver.Stream(), ver.MajorMinor(), imageKind) if err != nil { return err } @@ -255,9 +290,11 @@ type versionManager struct { bucket string distributionID string dirty bool // manager gets dirty on write + dryRun bool // no write operations + log *logger.Logger } -func newVersionManager(ctx context.Context, region, bucket, distributionID string) (*versionManager, error) { +func newVersionManager(ctx context.Context, region, bucket, distributionID string, dryRun bool, log *logger.Logger) (*versionManager, error) { cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) if err != nil { return nil, err @@ -272,6 +309,8 @@ func newVersionManager(ctx context.Context, region, bucket, distributionID strin uploader: uploader, bucket: bucket, distributionID: distributionID, + dryRun: dryRun, + log: log, }, nil } @@ -308,13 +347,20 @@ func (m *versionManager) updateVersionList(ctx context.Context, list *versionsap return err } - m.dirty = true - in := &s3.PutObjectInput{ Bucket: aws.String(m.bucket), Key: aws.String(listJSONPath(list)), Body: bytes.NewBuffer(rawList), } + + if m.dryRun { + m.log.Infof("dryRun: s3 put object {Bucket: %v, Key: %v, Body: %v", m.bucket, listJSONPath(list), string(rawList)) + + return nil + } + + m.dirty = true + _, err = m.uploader.Upload(ctx, in) return err @@ -369,7 +415,7 @@ func waitForCacheUpdate(ctx context.Context, updateFetcher *versionsapi.Fetcher, } func listJSONPath(list *versionsapi.List) string { - return path.Join(constants.CDNVersionsPath, "stream", list.Stream, list.Granularity, list.Base, imageKind+".json") + return path.Join(constants.CDNAPIPrefix, "ref", list.Ref, "stream", list.Stream, "versions", list.Granularity, list.Base, imageKind+".json") } type granularity int diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 3590195fa..a603c4fee 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -174,8 +174,8 @@ const ( CDNImagePath = "constellation/v1/images" // CDNMeasurementsPath is the default path to image measurements in the CDN repository. CDNMeasurementsPath = "constellation/v1/measurements" - // CDNVersionsPath is the default path to versions in the CDN repository. - CDNVersionsPath = "constellation/v1/versions" + // CDNAPIPrefix is the prefix for the Constellation CDN API. + CDNAPIPrefix = "constellation/v1" ) // VersionInfo is the version of a binary. Left as a separate variable to allow override during build. diff --git a/internal/versionsapi/versionsapi.go b/internal/versionsapi/versionsapi.go index 4fce30f9f..6f919de24 100644 --- a/internal/versionsapi/versionsapi.go +++ b/internal/versionsapi/versionsapi.go @@ -14,6 +14,7 @@ import ( "net/http" "net/url" "path" + "regexp" "strings" "github.com/edgelesssys/constellation/v2/internal/constants" @@ -28,23 +29,26 @@ import ( // A List with granularity "minor" could contain the base version // "v1.0" and a list of patch versions "v1.0.0", "v1.0.1", "v1.0.2" etc. type List struct { + // Ref is the branch name the list belongs to. + Ref string `json:"ref,omitempty"` // Stream is the update stream of the list. // Currently, only "stable" and "debug" are supported. - Stream string `json:"stream"` + Stream string `json:"stream,omitempty"` // Granularity is the granularity of the base version of this list. // It can be either "major" or "minor". - Granularity string `json:"granularity"` + Granularity string `json:"granularity,omitempty"` // Base is the base version of the list. // Every version in the list is a finer-grained version of this base version. - Base string `json:"base"` + Base string `json:"base,omitempty"` // Kind is the kind of resource this list is for. - Kind string `json:"kind"` + Kind string `json:"kind,omitempty"` // Versions is a list of all versions in this list. - Versions []string `json:"versions"` + Versions []string `json:"versions,omitempty"` } // Validate checks if the list is valid. // This performs the following checks: +// - The ref is set. // - The stream is supported. // - The granularity is "major" or "minor". // - The kind is supported. @@ -52,8 +56,11 @@ type List struct { // - All versions in the list are valid semantic versions that are finer-grained than the base version. func (l *List) Validate() error { var issues []string - if !IsValidStream(l.Stream) { - issues = append(issues, fmt.Sprintf("stream %q is not supported", l.Stream)) + if !IsValidRef(l.Ref) { + issues = append(issues, "ref is empty") + } + if !IsValidStream(l.Ref, l.Stream) { + issues = append(issues, fmt.Sprintf("stream %q is not supported on ref %q", l.Stream, l.Ref)) } if l.Granularity != "major" && l.Granularity != "minor" { issues = append(issues, fmt.Sprintf("granularity %q is not supported", l.Granularity)) @@ -113,18 +120,18 @@ func New() *Fetcher { } // MinorVersionsOf fetches the list of minor versions for a given stream, major version and kind. -func (f *Fetcher) MinorVersionsOf(ctx context.Context, stream, major, kind string) (*List, error) { - return f.list(ctx, stream, "major", major, kind) +func (f *Fetcher) MinorVersionsOf(ctx context.Context, ref, stream, major, kind string) (*List, error) { + return f.list(ctx, stream, "major", major, ref, kind) } // PatchVersionsOf fetches the list of patch versions for a given stream, minor version and kind. -func (f *Fetcher) PatchVersionsOf(ctx context.Context, stream, minor, kind string) (*List, error) { - return f.list(ctx, stream, "minor", minor, kind) +func (f *Fetcher) PatchVersionsOf(ctx context.Context, ref, stream, minor, kind string) (*List, error) { + return f.list(ctx, stream, "minor", minor, ref, kind) } // list fetches the list of versions for a given stream, granularity, base and kind. -func (f *Fetcher) list(ctx context.Context, stream, granularity, base, kind string) (*List, error) { - raw, err := getFromURL(ctx, f.httpc, stream, granularity, base, kind) +func (f *Fetcher) list(ctx context.Context, ref, stream, granularity, base, kind string) (*List, error) { + raw, err := getFromURL(ctx, f.httpc, ref, stream, granularity, base, kind) if err != nil { return nil, fmt.Errorf("fetching versions list: %w", err) } @@ -146,13 +153,13 @@ func (f *Fetcher) listMatchesRequest(list *List, stream, granularity, base, kind } // getFromURL fetches the versions list from a URL. -func getFromURL(ctx context.Context, client httpc, stream, granularity, base, kind string) ([]byte, error) { +func getFromURL(ctx context.Context, client httpc, ref, stream, granularity, base, kind string) ([]byte, error) { url, err := url.Parse(constants.CDNRepositoryURL) if err != nil { return nil, fmt.Errorf("parsing image version repository URL: %w", err) } kindFilename := path.Base(kind) + ".json" - url.Path = path.Join(constants.CDNVersionsPath, "stream", stream, granularity, base, kindFilename) + url.Path = path.Join(constants.CDNAPIPrefix, "ref", ref, "stream", stream, "versions", granularity, base, kindFilename) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), http.NoBody) if err != nil { return nil, err @@ -179,8 +186,13 @@ func getFromURL(ctx context.Context, client httpc, stream, granularity, base, ki } // IsValidStream returns true if the given stream is a valid stream. -func IsValidStream(stream string) bool { - validStreams := []string{"stable", "debug"} +func IsValidStream(ref, stream string) bool { + validReleaseStreams := []string{"stable", "console", "debug"} + validStreams := []string{"nightly", "console", "debug"} + + if isReleaseRef(ref) { + validStreams = validReleaseStreams + } for _, validStream := range validStreams { if stream == validStream { @@ -191,6 +203,34 @@ func IsValidStream(stream string) bool { return false } +var notAZ09Regexp = regexp.MustCompile("[^a-zA-Z0-9-]+") + +// CanonicalRef returns the canonicalized ref for the given ref. +func CanonicalRef(ref string) string { + return notAZ09Regexp.ReplaceAllString(ref, "-") +} + +// IsValidRef returns true if the given ref is a valid ref. +func IsValidRef(ref string) bool { + if ref == "" { + return false + } + + if notAZ09Regexp.FindString(ref) != "" { + return false + } + + if strings.HasPrefix(ref, "refs-heads") { + return false + } + + return true +} + +func isReleaseRef(ref string) bool { + return ref == "-" +} + type httpc interface { Do(req *http.Request) (*http.Response, error) } diff --git a/internal/versionsapi/versionsapi_test.go b/internal/versionsapi/versionsapi_test.go index 019941774..7c6a6f6ff 100644 --- a/internal/versionsapi/versionsapi_test.go +++ b/internal/versionsapi/versionsapi_test.go @@ -123,37 +123,37 @@ func TestList(t *testing.T) { require.NoError(t, err) client := newTestClient(func(req *http.Request) *http.Response { switch req.URL.Path { - case "/constellation/v1/versions/stream/stable/major/v1/image.json": + case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/image.json": return &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBuffer(majorListJSON)), Header: make(http.Header), } - case "/constellation/v1/versions/stream/stable/minor/v1.1/image.json": + case "/constellation/v1/ref/test-ref/stream/nightly/versions/minor/v1.1/image.json": return &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBuffer(minorListJSON)), Header: make(http.Header), } - case "/constellation/v1/versions/stream/stable/major/v1/500.json": // 500 error + case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/500.json": // 500 error return &http.Response{ StatusCode: http.StatusInternalServerError, Body: io.NopCloser(bytes.NewBufferString("Server Error.")), Header: make(http.Header), } - case "/constellation/v1/versions/stream/stable/major/v1/nojson.json": // invalid format + case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/nojson.json": // invalid format return &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBufferString("not json")), Header: make(http.Header), } - case "/constellation/v1/versions/stream/stable/major/v2/image.json": // inconsistent list + case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v2/image.json": // inconsistent list return &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBuffer(inconsistentListJSON)), Header: make(http.Header), } - case "/constellation/v1/versions/stream/stable/major/v3/image.json": // does not match requested version + case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v3/image.json": // does not match requested version return &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBuffer(minorListJSON)), @@ -168,10 +168,10 @@ func TestList(t *testing.T) { }) testCases := map[string]struct { - stream, granularity, base, kind string - overrideFile string - wantList List - wantErr bool + ref, stream, granularity, base, kind string + overrideFile string + wantList List + wantErr bool }{ "major list fetched remotely": { wantList: *majorList(), @@ -208,7 +208,8 @@ func TestList(t *testing.T) { assert := assert.New(t) require := require.New(t) - stream := "stable" + ref := "test-ref" + stream := "nightly" granularity := "major" base := "v1" kind := "image" @@ -228,7 +229,7 @@ func TestList(t *testing.T) { fetcher := &Fetcher{ httpc: client, } - list, err := fetcher.list(context.Background(), stream, granularity, base, kind) + list, err := fetcher.list(context.Background(), ref, stream, granularity, base, kind) if tc.wantErr { assert.Error(err) return @@ -256,7 +257,8 @@ func newTestClient(fn roundTripFunc) *http.Client { func majorList() *List { return &List{ - Stream: "stable", + Ref: "test-ref", + Stream: "nightly", Granularity: "major", Base: "v1", Kind: "image", @@ -268,7 +270,8 @@ func majorList() *List { func minorList() *List { return &List{ - Stream: "stable", + Ref: "test-ref", + Stream: "nightly", Granularity: "minor", Base: "v1.1", Kind: "image", @@ -278,21 +281,67 @@ func minorList() *List { } } -func TestIsValidStream(t *testing.T) { +func TestIsValidRef(t *testing.T) { testCases := map[string]bool{ - "stable": true, - "debug": true, - "beta": false, - "alpha": false, - "unknown": false, - "fast": false, + "feat/foo": false, + "feat-foo": true, + "feat$foo": false, + "3234": true, + "feat foo": false, + "refs-heads-feat-foo": false, + "": false, } - for name, want := range testCases { - t.Run(name, func(t *testing.T) { + for ref, want := range testCases { + t.Run(ref, func(t *testing.T) { assert := assert.New(t) - - assert.Equal(want, IsValidStream(name)) + assert.Equal(want, IsValidRef(ref)) + }) + } +} + +func TestCanonicalRef(t *testing.T) { + testCases := map[string]string{ + "feat/foo": "feat-foo", + "feat-foo": "feat-foo", + "feat$foo": "feat-foo", + "3234": "3234", + "feat foo": "feat-foo", + } + + for ref, want := range testCases { + t.Run(ref, func(t *testing.T) { + assert := assert.New(t) + assert.Equal(want, CanonicalRef(ref)) + }) + } +} + +func TestIsValidStream(t *testing.T) { + testCases := []struct { + branch string + stream string + want bool + }{ + {branch: "-", stream: "stable", want: true}, + {branch: "-", stream: "debug", want: true}, + {branch: "-", stream: "nightly", want: false}, + {branch: "-", stream: "console", want: true}, + {branch: "main", stream: "stable", want: false}, + {branch: "main", stream: "debug", want: true}, + {branch: "main", stream: "nightly", want: true}, + {branch: "main", stream: "console", want: true}, + {branch: "foo-branch", stream: "nightly", want: true}, + {branch: "foo-branch", stream: "console", want: true}, + {branch: "foo-branch", stream: "debug", want: true}, + {branch: "foo-branch", stream: "stable", want: false}, + } + + for _, tc := range testCases { + t.Run(tc.branch+"+"+tc.stream, func(t *testing.T) { + assert := assert.New(t) + + assert.Equal(tc.want, IsValidStream(tc.branch, tc.stream)) }) } }