hack: implement new api for add-version script

Signed-off-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>

cli/internal/cmd/upgradeplan.go

@@ -110,7 +110,7 @@ func upgradePlan(cmd *cobra.Command, planner upgradePlanner, patchLister patchLi
 
 	var updateCandidates []string
 	for _, minorVer := range allowedMinorVersions {
-		versionList, err := patchLister.PatchVersionsOf(cmd.Context(), "stable", minorVer, "image")
+		versionList, err := patchLister.PatchVersionsOf(cmd.Context(), "-", "stable", minorVer, "image")
 		if err == nil {
 			updateCandidates = append(updateCandidates, versionList.Versions...)
 		}
@@ -335,5 +335,5 @@ type upgradePlanner interface {
 }
 
 type patchLister interface {
-	PatchVersionsOf(ctx context.Context, stream, minor, kind string) (*versionsapi.List, error)
+	PatchVersionsOf(ctx context.Context, ref, stream, minor, kind string) (*versionsapi.List, error)
 }

cli/internal/cmd/upgradeplan_test.go

@@ -492,6 +492,6 @@ type stubPatchLister struct {
 	err  error
 }
 
-func (s stubPatchLister) PatchVersionsOf(ctx context.Context, stream, minor, kind string) (*versionsapi.List, error) {
+func (s stubPatchLister) PatchVersionsOf(ctx context.Context, ref, stream, minor, kind string) (*versionsapi.List, error) {
 	return &s.list, s.err
 }

hack/add-version/main.go

@@ -36,6 +36,7 @@ import (
 var errVersionListMissing = errors.New("version list does not exist")
 
 const (
+	skipRefStr                   = "-"
 	imageKind                    = "image"
 	defaultRegion                = "eu-central-1"
 	defaultBucket                = "cdn-constellation-backend"
@@ -50,6 +51,9 @@ func main() {
 	flags := flags{
 		version:        flag.String("version", "", "Version to add (format: \"v1.2.3\")"),
 		stream:         flag.String("stream", "", "Stream to add the version to"),
+		ref:            flag.String("ref", "", "Ref to add the version to"),
+		release:        flag.Bool("release", false, "Whether the version is a release"),
+		dryRun:         flag.Bool("dryrun", false, "Whether to run in dry-run mode (no changes are made)"),
 		region:         flag.String("region", defaultRegion, "AWS region"),
 		bucket:         flag.String("bucket", defaultBucket, "S3 bucket"),
 		distributionID: flag.String("distribution-id", defaultDistributionID, "cloudfront distribution id"),
@@ -60,7 +64,7 @@ func main() {
 	}
 
 	updateFetcher := versionsapi.New()
-	versionManager, err := newVersionManager(ctx, *flags.region, *flags.bucket, *flags.distributionID)
+	versionManager, err := newVersionManager(ctx, *flags.region, *flags.bucket, *flags.distributionID, *flags.dryRun, log)
 	if err != nil {
 		log.With(zap.Error(err)).Fatalf("Failed to create version uploader")
 	}
@@ -68,6 +72,7 @@ func main() {
 	ver := version{
 		versionStr: *flags.version,
 		stream:     *flags.stream,
+		ref:        *flags.ref,
 	}
 
 	if err := ensureMinorVersion(ctx, versionManager, ver, log); err != nil {
@@ -101,6 +106,7 @@ func ensureMinorVersion(ctx context.Context, versionManager *versionManager, ver
 	if errors.Is(err, errVersionListMissing) {
 		log.Infof("Version list for minor versions under %q does not exist. Creating new list.", ver.Major())
 		minorVerList = &versionsapi.List{
+			Ref:         ver.Ref(),
 			Stream:      ver.Stream(),
 			Granularity: "major",
 			Base:        ver.Major(),
@@ -132,6 +138,7 @@ func ensurePatchVersion(ctx context.Context, versionManager *versionManager, ver
 	if errors.Is(err, errVersionListMissing) {
 		log.Infof("Version list for patch versions under %q does not exist. Creating new list.", ver.MajorMinor())
 		pathVerList = &versionsapi.List{
+			Ref:         ver.Ref(),
 			Stream:      ver.Stream(),
 			Granularity: "minor",
 			Base:        ver.MajorMinor(),
@@ -160,6 +167,7 @@ func ensurePatchVersion(ctx context.Context, versionManager *versionManager, ver
 type version struct {
 	versionStr string
 	stream     string
+	ref        string
 }
 
 func (v *version) MajorMinorPatch() string {
@@ -190,16 +198,23 @@ func (v *version) URL(gran granularity) string {
 }
 
 func (v *version) JSONPath(gran granularity) string {
-	return path.Join(constants.CDNVersionsPath, "stream", v.stream, gran.String(), v.WithGranularity(gran), imageKind+".json")
+	return path.Join(constants.CDNAPIPrefix, "ref", v.ref, "stream", v.stream, "versions", gran.String(), v.WithGranularity(gran), imageKind+".json")
 }
 
 func (v *version) Stream() string {
 	return v.stream
 }
 
+func (v *version) Ref() string {
+	return v.ref
+}
+
 type flags struct {
 	version        *string
 	stream         *string
+	ref            *string
+	release        *bool
+	dryRun         *bool
 	region         *string
 	bucket         *string
 	distributionID *string
@@ -209,9 +224,29 @@ func (f *flags) validate() error {
 	if err := validateVersion(*f.version); err != nil {
 		return err
 	}
-	if *f.stream == "" {
+
-		return errors.New("stream must be set")
+	if *f.ref == "" && !*f.release {
+		if !*f.release {
+			return fmt.Errorf("branch flag must be set for non-release versions")
+		}
 	}
+	if *f.ref != "" && *f.release {
+		return fmt.Errorf("branch flag must not be set for release versions")
+	}
+	if *f.release {
+		*f.ref = skipRefStr
+	}
+
+	ref := versionsapi.CanonicalRef(*f.ref)
+	if !versionsapi.IsValidRef(ref) {
+		return fmt.Errorf("invalid ref %q", *f.ref)
+	}
+	*f.ref = ref
+
+	if !versionsapi.IsValidStream(*f.ref, *f.stream) {
+		return fmt.Errorf("invalid stream %q for ref %q", *f.stream, *f.ref)
+	}
+
 	return nil
 }
 
@@ -226,7 +261,7 @@ func validateVersion(version string) error {
 }
 
 func ensureMinorVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher, ver version) error {
-	existingMinorVersions, err := fetcher.MinorVersionsOf(ctx, ver.Stream(), ver.Major(), imageKind)
+	existingMinorVersions, err := fetcher.MinorVersionsOf(ctx, ver.Ref(), ver.Stream(), ver.Major(), imageKind)
 	if err != nil {
 		return err
 	}
@@ -237,7 +272,7 @@ func ensureMinorVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher,
 }
 
 func ensurePatchVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher, ver version) error {
-	existingPatchVersions, err := fetcher.PatchVersionsOf(ctx, ver.Stream(), ver.MajorMinor(), imageKind)
+	existingPatchVersions, err := fetcher.PatchVersionsOf(ctx, ver.Ref(), ver.Stream(), ver.MajorMinor(), imageKind)
 	if err != nil {
 		return err
 	}
@@ -255,9 +290,11 @@ type versionManager struct {
 	bucket         string
 	distributionID string
 	dirty          bool // manager gets dirty on write
+	dryRun         bool // no write operations
+	log            *logger.Logger
 }
 
-func newVersionManager(ctx context.Context, region, bucket, distributionID string) (*versionManager, error) {
+func newVersionManager(ctx context.Context, region, bucket, distributionID string, dryRun bool, log *logger.Logger) (*versionManager, error) {
 	cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region))
 	if err != nil {
 		return nil, err
@@ -272,6 +309,8 @@ func newVersionManager(ctx context.Context, region, bucket, distributionID strin
 		uploader:       uploader,
 		bucket:         bucket,
 		distributionID: distributionID,
+		dryRun:         dryRun,
+		log:            log,
 	}, nil
 }
 
@@ -308,13 +347,20 @@ func (m *versionManager) updateVersionList(ctx context.Context, list *versionsap
 		return err
 	}
 
-	m.dirty = true
-
 	in := &s3.PutObjectInput{
 		Bucket: aws.String(m.bucket),
 		Key:    aws.String(listJSONPath(list)),
 		Body:   bytes.NewBuffer(rawList),
 	}
+
+	if m.dryRun {
+		m.log.Infof("dryRun: s3 put object {Bucket: %v, Key: %v, Body: %v", m.bucket, listJSONPath(list), string(rawList))
+
+		return nil
+	}
+
+	m.dirty = true
+
 	_, err = m.uploader.Upload(ctx, in)
 
 	return err
@@ -369,7 +415,7 @@ func waitForCacheUpdate(ctx context.Context, updateFetcher *versionsapi.Fetcher,
 }
 
 func listJSONPath(list *versionsapi.List) string {
-	return path.Join(constants.CDNVersionsPath, "stream", list.Stream, list.Granularity, list.Base, imageKind+".json")
+	return path.Join(constants.CDNAPIPrefix, "ref", list.Ref, "stream", list.Stream, "versions", list.Granularity, list.Base, imageKind+".json")
 }
 
 type granularity int

internal/constants/constants.go

@@ -174,8 +174,8 @@ const (
 	CDNImagePath = "constellation/v1/images"
 	// CDNMeasurementsPath is the default path to image measurements in the CDN repository.
 	CDNMeasurementsPath = "constellation/v1/measurements"
-	// CDNVersionsPath is the default path to versions in the CDN repository.
+	// CDNAPIPrefix is the prefix for the Constellation CDN API.
-	CDNVersionsPath = "constellation/v1/versions"
+	CDNAPIPrefix = "constellation/v1"
 )
 
 // VersionInfo is the version of a binary. Left as a separate variable to allow override during build.

internal/versionsapi/versionsapi.go

@@ -14,6 +14,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"regexp"
 	"strings"
 
 	"github.com/edgelesssys/constellation/v2/internal/constants"
@@ -28,23 +29,26 @@ import (
 // A List with granularity "minor" could contain the base version
 // "v1.0" and a list of patch versions "v1.0.0", "v1.0.1", "v1.0.2" etc.
 type List struct {
+	// Ref is the branch name the list belongs to.
+	Ref string `json:"ref,omitempty"`
 	// Stream is the update stream of the list.
 	// Currently, only "stable" and "debug" are supported.
-	Stream string `json:"stream"`
+	Stream string `json:"stream,omitempty"`
 	// Granularity is the granularity of the base version of this list.
 	// It can be either "major" or "minor".
-	Granularity string `json:"granularity"`
+	Granularity string `json:"granularity,omitempty"`
 	// Base is the base version of the list.
 	// Every version in the list is a finer-grained version of this base version.
-	Base string `json:"base"`
+	Base string `json:"base,omitempty"`
 	// Kind is the kind of resource this list is for.
-	Kind string `json:"kind"`
+	Kind string `json:"kind,omitempty"`
 	// Versions is a list of all versions in this list.
-	Versions []string `json:"versions"`
+	Versions []string `json:"versions,omitempty"`
 }
 
 // Validate checks if the list is valid.
 // This performs the following checks:
+// - The ref is set.
 // - The stream is supported.
 // - The granularity is "major" or "minor".
 // - The kind is supported.
@@ -52,8 +56,11 @@ type List struct {
 // - All versions in the list are valid semantic versions that are finer-grained than the base version.
 func (l *List) Validate() error {
 	var issues []string
-	if !IsValidStream(l.Stream) {
+	if !IsValidRef(l.Ref) {
-		issues = append(issues, fmt.Sprintf("stream %q is not supported", l.Stream))
+		issues = append(issues, "ref is empty")
+	}
+	if !IsValidStream(l.Ref, l.Stream) {
+		issues = append(issues, fmt.Sprintf("stream %q is not supported on ref %q", l.Stream, l.Ref))
 	}
 	if l.Granularity != "major" && l.Granularity != "minor" {
 		issues = append(issues, fmt.Sprintf("granularity %q is not supported", l.Granularity))
@@ -113,18 +120,18 @@ func New() *Fetcher {
 }
 
 // MinorVersionsOf fetches the list of minor versions for a given stream, major version and kind.
-func (f *Fetcher) MinorVersionsOf(ctx context.Context, stream, major, kind string) (*List, error) {
+func (f *Fetcher) MinorVersionsOf(ctx context.Context, ref, stream, major, kind string) (*List, error) {
-	return f.list(ctx, stream, "major", major, kind)
+	return f.list(ctx, stream, "major", major, ref, kind)
 }
 
 // PatchVersionsOf fetches the list of patch versions for a given stream, minor version and kind.
-func (f *Fetcher) PatchVersionsOf(ctx context.Context, stream, minor, kind string) (*List, error) {
+func (f *Fetcher) PatchVersionsOf(ctx context.Context, ref, stream, minor, kind string) (*List, error) {
-	return f.list(ctx, stream, "minor", minor, kind)
+	return f.list(ctx, stream, "minor", minor, ref, kind)
 }
 
 // list fetches the list of versions for a given stream, granularity, base and kind.
-func (f *Fetcher) list(ctx context.Context, stream, granularity, base, kind string) (*List, error) {
+func (f *Fetcher) list(ctx context.Context, ref, stream, granularity, base, kind string) (*List, error) {
-	raw, err := getFromURL(ctx, f.httpc, stream, granularity, base, kind)
+	raw, err := getFromURL(ctx, f.httpc, ref, stream, granularity, base, kind)
 	if err != nil {
 		return nil, fmt.Errorf("fetching versions list: %w", err)
 	}
@@ -146,13 +153,13 @@ func (f *Fetcher) listMatchesRequest(list *List, stream, granularity, base, kind
 }
 
 // getFromURL fetches the versions list from a URL.
-func getFromURL(ctx context.Context, client httpc, stream, granularity, base, kind string) ([]byte, error) {
+func getFromURL(ctx context.Context, client httpc, ref, stream, granularity, base, kind string) ([]byte, error) {
 	url, err := url.Parse(constants.CDNRepositoryURL)
 	if err != nil {
 		return nil, fmt.Errorf("parsing image version repository URL: %w", err)
 	}
 	kindFilename := path.Base(kind) + ".json"
-	url.Path = path.Join(constants.CDNVersionsPath, "stream", stream, granularity, base, kindFilename)
+	url.Path = path.Join(constants.CDNAPIPrefix, "ref", ref, "stream", stream, "versions", granularity, base, kindFilename)
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), http.NoBody)
 	if err != nil {
 		return nil, err
@@ -179,8 +186,13 @@ func getFromURL(ctx context.Context, client httpc, stream, granularity, base, ki
 }
 
 // IsValidStream returns true if the given stream is a valid stream.
-func IsValidStream(stream string) bool {
+func IsValidStream(ref, stream string) bool {
-	validStreams := []string{"stable", "debug"}
+	validReleaseStreams := []string{"stable", "console", "debug"}
+	validStreams := []string{"nightly", "console", "debug"}
+
+	if isReleaseRef(ref) {
+		validStreams = validReleaseStreams
+	}
 
 	for _, validStream := range validStreams {
 		if stream == validStream {
@@ -191,6 +203,34 @@ func IsValidStream(stream string) bool {
 	return false
 }
 
+var notAZ09Regexp = regexp.MustCompile("[^a-zA-Z0-9-]+")
+
+// CanonicalRef returns the canonicalized ref for the given ref.
+func CanonicalRef(ref string) string {
+	return notAZ09Regexp.ReplaceAllString(ref, "-")
+}
+
+// IsValidRef returns true if the given ref is a valid ref.
+func IsValidRef(ref string) bool {
+	if ref == "" {
+		return false
+	}
+
+	if notAZ09Regexp.FindString(ref) != "" {
+		return false
+	}
+
+	if strings.HasPrefix(ref, "refs-heads") {
+		return false
+	}
+
+	return true
+}
+
+func isReleaseRef(ref string) bool {
+	return ref == "-"
+}
+
 type httpc interface {
 	Do(req *http.Request) (*http.Response, error)
 }

internal/versionsapi/versionsapi_test.go

@@ -123,37 +123,37 @@ func TestList(t *testing.T) {
 	require.NoError(t, err)
 	client := newTestClient(func(req *http.Request) *http.Response {
 		switch req.URL.Path {
-		case "/constellation/v1/versions/stream/stable/major/v1/image.json":
+		case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/image.json":
 			return &http.Response{
 				StatusCode: http.StatusOK,
 				Body:       io.NopCloser(bytes.NewBuffer(majorListJSON)),
 				Header:     make(http.Header),
 			}
-		case "/constellation/v1/versions/stream/stable/minor/v1.1/image.json":
+		case "/constellation/v1/ref/test-ref/stream/nightly/versions/minor/v1.1/image.json":
 			return &http.Response{
 				StatusCode: http.StatusOK,
 				Body:       io.NopCloser(bytes.NewBuffer(minorListJSON)),
 				Header:     make(http.Header),
 			}
-		case "/constellation/v1/versions/stream/stable/major/v1/500.json": // 500 error
+		case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/500.json": // 500 error
 			return &http.Response{
 				StatusCode: http.StatusInternalServerError,
 				Body:       io.NopCloser(bytes.NewBufferString("Server Error.")),
 				Header:     make(http.Header),
 			}
-		case "/constellation/v1/versions/stream/stable/major/v1/nojson.json": // invalid format
+		case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/nojson.json": // invalid format
 			return &http.Response{
 				StatusCode: http.StatusOK,
 				Body:       io.NopCloser(bytes.NewBufferString("not json")),
 				Header:     make(http.Header),
 			}
-		case "/constellation/v1/versions/stream/stable/major/v2/image.json": // inconsistent list
+		case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v2/image.json": // inconsistent list
 			return &http.Response{
 				StatusCode: http.StatusOK,
 				Body:       io.NopCloser(bytes.NewBuffer(inconsistentListJSON)),
 				Header:     make(http.Header),
 			}
-		case "/constellation/v1/versions/stream/stable/major/v3/image.json": // does not match requested version
+		case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v3/image.json": // does not match requested version
 			return &http.Response{
 				StatusCode: http.StatusOK,
 				Body:       io.NopCloser(bytes.NewBuffer(minorListJSON)),
@@ -168,10 +168,10 @@ func TestList(t *testing.T) {
 	})
 
 	testCases := map[string]struct {
-		stream, granularity, base, kind string
+		ref, stream, granularity, base, kind string
-		overrideFile                    string
+		overrideFile                         string
-		wantList                        List
+		wantList                             List
-		wantErr                         bool
+		wantErr                              bool
 	}{
 		"major list fetched remotely": {
 			wantList: *majorList(),
@@ -208,7 +208,8 @@ func TestList(t *testing.T) {
 			assert := assert.New(t)
 			require := require.New(t)
 
-			stream := "stable"
+			ref := "test-ref"
+			stream := "nightly"
 			granularity := "major"
 			base := "v1"
 			kind := "image"
@@ -228,7 +229,7 @@ func TestList(t *testing.T) {
 			fetcher := &Fetcher{
 				httpc: client,
 			}
-			list, err := fetcher.list(context.Background(), stream, granularity, base, kind)
+			list, err := fetcher.list(context.Background(), ref, stream, granularity, base, kind)
 			if tc.wantErr {
 				assert.Error(err)
 				return
@@ -256,7 +257,8 @@ func newTestClient(fn roundTripFunc) *http.Client {
 
 func majorList() *List {
 	return &List{
-		Stream:      "stable",
+		Ref:         "test-ref",
+		Stream:      "nightly",
 		Granularity: "major",
 		Base:        "v1",
 		Kind:        "image",
@@ -268,7 +270,8 @@ func majorList() *List {
 
 func minorList() *List {
 	return &List{
-		Stream:      "stable",
+		Ref:         "test-ref",
+		Stream:      "nightly",
 		Granularity: "minor",
 		Base:        "v1.1",
 		Kind:        "image",
@@ -278,21 +281,67 @@ func minorList() *List {
 	}
 }
 
-func TestIsValidStream(t *testing.T) {
+func TestIsValidRef(t *testing.T) {
 	testCases := map[string]bool{
-		"stable":  true,
+		"feat/foo":            false,
-		"debug":   true,
+		"feat-foo":            true,
-		"beta":    false,
+		"feat$foo":            false,
-		"alpha":   false,
+		"3234":                true,
-		"unknown": false,
+		"feat foo":            false,
-		"fast":    false,
+		"refs-heads-feat-foo": false,
+		"":                    false,
 	}
 
-	for name, want := range testCases {
+	for ref, want := range testCases {
-		t.Run(name, func(t *testing.T) {
+		t.Run(ref, func(t *testing.T) {
 			assert := assert.New(t)
-
+			assert.Equal(want, IsValidRef(ref))
-			assert.Equal(want, IsValidStream(name))
+		})
+	}
+}
+
+func TestCanonicalRef(t *testing.T) {
+	testCases := map[string]string{
+		"feat/foo": "feat-foo",
+		"feat-foo": "feat-foo",
+		"feat$foo": "feat-foo",
+		"3234":     "3234",
+		"feat foo": "feat-foo",
+	}
+
+	for ref, want := range testCases {
+		t.Run(ref, func(t *testing.T) {
+			assert := assert.New(t)
+			assert.Equal(want, CanonicalRef(ref))
+		})
+	}
+}
+
+func TestIsValidStream(t *testing.T) {
+	testCases := []struct {
+		branch string
+		stream string
+		want   bool
+	}{
+		{branch: "-", stream: "stable", want: true},
+		{branch: "-", stream: "debug", want: true},
+		{branch: "-", stream: "nightly", want: false},
+		{branch: "-", stream: "console", want: true},
+		{branch: "main", stream: "stable", want: false},
+		{branch: "main", stream: "debug", want: true},
+		{branch: "main", stream: "nightly", want: true},
+		{branch: "main", stream: "console", want: true},
+		{branch: "foo-branch", stream: "nightly", want: true},
+		{branch: "foo-branch", stream: "console", want: true},
+		{branch: "foo-branch", stream: "debug", want: true},
+		{branch: "foo-branch", stream: "stable", want: false},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.branch+"+"+tc.stream, func(t *testing.T) {
+			assert := assert.New(t)
+
+			assert.Equal(tc.want, IsValidStream(tc.branch, tc.stream))
 		})
 	}
 }