Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-12-24 15:09:39 -05:00
Code Issues Packages Projects Releases Wiki Activity

hack: implement new api for add-version script

Browse Source 
Signed-off-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>
This commit is contained in:
Paul Meyer 2022-12-05 15:15:03 +01:00 committed by Malte Poll
parent e461b6385a
commit f23a2fe073
6 changed files with 192 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cli/internal/cmd/upgradeplan.go
View File

@ -110,7 +110,7 @@ func upgradePlan(cmd *cobra.Command, planner upgradePlanner, patchLister patchLi
var updateCandidates []string
for _, minorVer := range allowedMinorVersions {
versionList, err := patchLister.PatchVersionsOf(cmd.Context(), "stable", minorVer, "image")
versionList, err := patchLister.PatchVersionsOf(cmd.Context(), "-", "stable", minorVer, "image")
if err == nil {
updateCandidates = append(updateCandidates, versionList.Versions...)
}
@ -335,5 +335,5 @@ type upgradePlanner interface {
}
type patchLister interface {
PatchVersionsOf(ctx context.Context, stream, minor, kind string) (*versionsapi.List, error)
PatchVersionsOf(ctx context.Context, ref, stream, minor, kind string) (*versionsapi.List, error)
}

2
cli/internal/cmd/upgradeplan_test.go
View File

@ -492,6 +492,6 @@ type stubPatchLister struct {
err error
}
func (s stubPatchLister) PatchVersionsOf(ctx context.Context, stream, minor, kind string) (*versionsapi.List, error) {
func (s stubPatchLister) PatchVersionsOf(ctx context.Context, ref, stream, minor, kind string) (*versionsapi.List, error) {
return &s.list, s.err
}

66
hack/add-version/main.go
View File

@ -36,6 +36,7 @@ import (
var errVersionListMissing = errors.New("version list does not exist")
const (
skipRefStr = "-"
imageKind = "image"
defaultRegion = "eu-central-1"
defaultBucket = "cdn-constellation-backend"
@ -50,6 +51,9 @@ func main() {
flags := flags{
version: flag.String("version", "", "Version to add (format: \"v1.2.3\")"),
stream: flag.String("stream", "", "Stream to add the version to"),
ref: flag.String("ref", "", "Ref to add the version to"),
release: flag.Bool("release", false, "Whether the version is a release"),
dryRun: flag.Bool("dryrun", false, "Whether to run in dry-run mode (no changes are made)"),
region: flag.String("region", defaultRegion, "AWS region"),
bucket: flag.String("bucket", defaultBucket, "S3 bucket"),
distributionID: flag.String("distribution-id", defaultDistributionID, "cloudfront distribution id"),
@ -60,7 +64,7 @@ func main() {
}
updateFetcher := versionsapi.New()
versionManager, err := newVersionManager(ctx, *flags.region, *flags.bucket, *flags.distributionID)
versionManager, err := newVersionManager(ctx, *flags.region, *flags.bucket, *flags.distributionID, *flags.dryRun, log)
if err != nil {
log.With(zap.Error(err)).Fatalf("Failed to create version uploader")
}
@ -68,6 +72,7 @@ func main() {
ver := version{
versionStr: *flags.version,
stream: *flags.stream,
ref: *flags.ref,
}
if err := ensureMinorVersion(ctx, versionManager, ver, log); err != nil {
@ -101,6 +106,7 @@ func ensureMinorVersion(ctx context.Context, versionManager *versionManager, ver
if errors.Is(err, errVersionListMissing) {
log.Infof("Version list for minor versions under %q does not exist. Creating new list.", ver.Major())
minorVerList = &versionsapi.List{
Ref: ver.Ref(),
Stream: ver.Stream(),
Granularity: "major",
Base: ver.Major(),
@ -132,6 +138,7 @@ func ensurePatchVersion(ctx context.Context, versionManager *versionManager, ver
if errors.Is(err, errVersionListMissing) {
log.Infof("Version list for patch versions under %q does not exist. Creating new list.", ver.MajorMinor())
pathVerList = &versionsapi.List{
Ref: ver.Ref(),
Stream: ver.Stream(),
Granularity: "minor",
Base: ver.MajorMinor(),
@ -160,6 +167,7 @@ func ensurePatchVersion(ctx context.Context, versionManager *versionManager, ver
type version struct {
versionStr string
stream string
ref string
}
func (v *version) MajorMinorPatch() string {
@ -190,16 +198,23 @@ func (v *version) URL(gran granularity) string {
}
func (v *version) JSONPath(gran granularity) string {
return path.Join(constants.CDNVersionsPath, "stream", v.stream, gran.String(), v.WithGranularity(gran), imageKind+".json")
return path.Join(constants.CDNAPIPrefix, "ref", v.ref, "stream", v.stream, "versions", gran.String(), v.WithGranularity(gran), imageKind+".json")
}
func (v *version) Stream() string {
return v.stream
}
func (v *version) Ref() string {
return v.ref
}
type flags struct {
version *string
stream *string
ref *string
release *bool
dryRun *bool
region *string
bucket *string
distributionID *string
@ -209,9 +224,29 @@ func (f *flags) validate() error {
if err := validateVersion(*f.version); err != nil {
return err
}
if *f.stream == "" {
return errors.New("stream must be set")
if *f.ref == "" && !*f.release {
if !*f.release {
return fmt.Errorf("branch flag must be set for non-release versions")
}
}
if *f.ref != "" && *f.release {
return fmt.Errorf("branch flag must not be set for release versions")
}
if *f.release {
*f.ref = skipRefStr
}
ref := versionsapi.CanonicalRef(*f.ref)
if !versionsapi.IsValidRef(ref) {
return fmt.Errorf("invalid ref %q", *f.ref)
}
*f.ref = ref
if !versionsapi.IsValidStream(*f.ref, *f.stream) {
return fmt.Errorf("invalid stream %q for ref %q", *f.stream, *f.ref)
}
return nil
}
@ -226,7 +261,7 @@ func validateVersion(version string) error {
}
func ensureMinorVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher, ver version) error {
existingMinorVersions, err := fetcher.MinorVersionsOf(ctx, ver.Stream(), ver.Major(), imageKind)
existingMinorVersions, err := fetcher.MinorVersionsOf(ctx, ver.Ref(), ver.Stream(), ver.Major(), imageKind)
if err != nil {
return err
}
@ -237,7 +272,7 @@ func ensureMinorVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher,
}
func ensurePatchVersionExists(ctx context.Context, fetcher *versionsapi.Fetcher, ver version) error {
existingPatchVersions, err := fetcher.PatchVersionsOf(ctx, ver.Stream(), ver.MajorMinor(), imageKind)
existingPatchVersions, err := fetcher.PatchVersionsOf(ctx, ver.Ref(), ver.Stream(), ver.MajorMinor(), imageKind)
if err != nil {
return err
}
@ -255,9 +290,11 @@ type versionManager struct {
bucket string
distributionID string
dirty bool // manager gets dirty on write
dryRun bool // no write operations
log *logger.Logger
}
func newVersionManager(ctx context.Context, region, bucket, distributionID string) (*versionManager, error) {
func newVersionManager(ctx context.Context, region, bucket, distributionID string, dryRun bool, log *logger.Logger) (*versionManager, error) {
cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region))
if err != nil {
return nil, err
@ -272,6 +309,8 @@ func newVersionManager(ctx context.Context, region, bucket, distributionID strin
uploader: uploader,
bucket: bucket,
distributionID: distributionID,
dryRun: dryRun,
log: log,
}, nil
}
@ -308,13 +347,20 @@ func (m *versionManager) updateVersionList(ctx context.Context, list *versionsap
return err
}
m.dirty = true
in := &s3.PutObjectInput{
Bucket: aws.String(m.bucket),
Key: aws.String(listJSONPath(list)),
Body: bytes.NewBuffer(rawList),
}
if m.dryRun {
m.log.Infof("dryRun: s3 put object {Bucket: %v, Key: %v, Body: %v", m.bucket, listJSONPath(list), string(rawList))
return nil
}
m.dirty = true
_, err = m.uploader.Upload(ctx, in)
return err
@ -369,7 +415,7 @@ func waitForCacheUpdate(ctx context.Context, updateFetcher *versionsapi.Fetcher,
}
func listJSONPath(list *versionsapi.List) string {
return path.Join(constants.CDNVersionsPath, "stream", list.Stream, list.Granularity, list.Base, imageKind+".json")
return path.Join(constants.CDNAPIPrefix, "ref", list.Ref, "stream", list.Stream, "versions", list.Granularity, list.Base, imageKind+".json")
}
type granularity int

4
internal/constants/constants.go
View File

@ -174,8 +174,8 @@ const (
CDNImagePath = "constellation/v1/images"
// CDNMeasurementsPath is the default path to image measurements in the CDN repository.
CDNMeasurementsPath = "constellation/v1/measurements"
// CDNVersionsPath is the default path to versions in the CDN repository.
CDNVersionsPath = "constellation/v1/versions"
// CDNAPIPrefix is the prefix for the Constellation CDN API.
CDNAPIPrefix = "constellation/v1"
)
// VersionInfo is the version of a binary. Left as a separate variable to allow override during build.

74
internal/versionsapi/versionsapi.go
View File

@ -14,6 +14,7 @@ import (
"net/http"
"net/url"
"path"
"regexp"
"strings"
"github.com/edgelesssys/constellation/v2/internal/constants"
@ -28,23 +29,26 @@ import (
// A List with granularity "minor" could contain the base version
// "v1.0" and a list of patch versions "v1.0.0", "v1.0.1", "v1.0.2" etc.
type List struct {
// Ref is the branch name the list belongs to.
Ref string `json:"ref,omitempty"`
// Stream is the update stream of the list.
// Currently, only "stable" and "debug" are supported.
Stream string `json:"stream"`
Stream string `json:"stream,omitempty"`
// Granularity is the granularity of the base version of this list.
// It can be either "major" or "minor".
Granularity string `json:"granularity"`
Granularity string `json:"granularity,omitempty"`
// Base is the base version of the list.
// Every version in the list is a finer-grained version of this base version.
Base string `json:"base"`
Base string `json:"base,omitempty"`
// Kind is the kind of resource this list is for.
Kind string `json:"kind"`
Kind string `json:"kind,omitempty"`
// Versions is a list of all versions in this list.
Versions []string `json:"versions"`
Versions []string `json:"versions,omitempty"`
}
// Validate checks if the list is valid.
// This performs the following checks:
// - The ref is set.
// - The stream is supported.
// - The granularity is "major" or "minor".
// - The kind is supported.
@ -52,8 +56,11 @@ type List struct {
// - All versions in the list are valid semantic versions that are finer-grained than the base version.
func (l *List) Validate() error {
var issues []string
if !IsValidStream(l.Stream) {
issues = append(issues, fmt.Sprintf("stream %q is not supported", l.Stream))
if !IsValidRef(l.Ref) {
issues = append(issues, "ref is empty")
}
if !IsValidStream(l.Ref, l.Stream) {
issues = append(issues, fmt.Sprintf("stream %q is not supported on ref %q", l.Stream, l.Ref))
}
if l.Granularity != "major" && l.Granularity != "minor" {
issues = append(issues, fmt.Sprintf("granularity %q is not supported", l.Granularity))
@ -113,18 +120,18 @@ func New() *Fetcher {
}
// MinorVersionsOf fetches the list of minor versions for a given stream, major version and kind.
func (f *Fetcher) MinorVersionsOf(ctx context.Context, stream, major, kind string) (*List, error) {
return f.list(ctx, stream, "major", major, kind)
func (f *Fetcher) MinorVersionsOf(ctx context.Context, ref, stream, major, kind string) (*List, error) {
return f.list(ctx, stream, "major", major, ref, kind)
}
// PatchVersionsOf fetches the list of patch versions for a given stream, minor version and kind.
func (f *Fetcher) PatchVersionsOf(ctx context.Context, stream, minor, kind string) (*List, error) {
return f.list(ctx, stream, "minor", minor, kind)
func (f *Fetcher) PatchVersionsOf(ctx context.Context, ref, stream, minor, kind string) (*List, error) {
return f.list(ctx, stream, "minor", minor, ref, kind)
}
// list fetches the list of versions for a given stream, granularity, base and kind.
func (f *Fetcher) list(ctx context.Context, stream, granularity, base, kind string) (*List, error) {
raw, err := getFromURL(ctx, f.httpc, stream, granularity, base, kind)
func (f *Fetcher) list(ctx context.Context, ref, stream, granularity, base, kind string) (*List, error) {
raw, err := getFromURL(ctx, f.httpc, ref, stream, granularity, base, kind)
if err != nil {
return nil, fmt.Errorf("fetching versions list: %w", err)
}
@ -146,13 +153,13 @@ func (f *Fetcher) listMatchesRequest(list *List, stream, granularity, base, kind
}
// getFromURL fetches the versions list from a URL.
func getFromURL(ctx context.Context, client httpc, stream, granularity, base, kind string) ([]byte, error) {
func getFromURL(ctx context.Context, client httpc, ref, stream, granularity, base, kind string) ([]byte, error) {
url, err := url.Parse(constants.CDNRepositoryURL)
if err != nil {
return nil, fmt.Errorf("parsing image version repository URL: %w", err)
}
kindFilename := path.Base(kind) + ".json"
url.Path = path.Join(constants.CDNVersionsPath, "stream", stream, granularity, base, kindFilename)
url.Path = path.Join(constants.CDNAPIPrefix, "ref", ref, "stream", stream, "versions", granularity, base, kindFilename)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), http.NoBody)
if err != nil {
return nil, err
@ -179,8 +186,13 @@ func getFromURL(ctx context.Context, client httpc, stream, granularity, base, ki
}
// IsValidStream returns true if the given stream is a valid stream.
func IsValidStream(stream string) bool {
validStreams := []string{"stable", "debug"}
func IsValidStream(ref, stream string) bool {
validReleaseStreams := []string{"stable", "console", "debug"}
validStreams := []string{"nightly", "console", "debug"}
if isReleaseRef(ref) {
validStreams = validReleaseStreams
}
for _, validStream := range validStreams {
if stream == validStream {
@ -191,6 +203,34 @@ func IsValidStream(stream string) bool {
return false
}
var notAZ09Regexp = regexp.MustCompile("[^a-zA-Z0-9-]+")
// CanonicalRef returns the canonicalized ref for the given ref.
func CanonicalRef(ref string) string {
return notAZ09Regexp.ReplaceAllString(ref, "-")
}
// IsValidRef returns true if the given ref is a valid ref.
func IsValidRef(ref string) bool {
if ref == "" {
return false
}
if notAZ09Regexp.FindString(ref) != "" {
return false
}
if strings.HasPrefix(ref, "refs-heads") {
return false
}
return true
}
func isReleaseRef(ref string) bool {
return ref == "-"
}
type httpc interface {
Do(req *http.Request) (*http.Response, error)
}

99
internal/versionsapi/versionsapi_test.go
View File

@ -123,37 +123,37 @@ func TestList(t *testing.T) {
require.NoError(t, err)
client := newTestClient(func(req *http.Request) *http.Response {
switch req.URL.Path {
case "/constellation/v1/versions/stream/stable/major/v1/image.json":
case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/image.json":
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(bytes.NewBuffer(majorListJSON)),
Header: make(http.Header),
}
case "/constellation/v1/versions/stream/stable/minor/v1.1/image.json":
case "/constellation/v1/ref/test-ref/stream/nightly/versions/minor/v1.1/image.json":
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(bytes.NewBuffer(minorListJSON)),
Header: make(http.Header),
}
case "/constellation/v1/versions/stream/stable/major/v1/500.json": // 500 error
case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/500.json": // 500 error
return &http.Response{
StatusCode: http.StatusInternalServerError,
Body: io.NopCloser(bytes.NewBufferString("Server Error.")),
Header: make(http.Header),
}
case "/constellation/v1/versions/stream/stable/major/v1/nojson.json": // invalid format
case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v1/nojson.json": // invalid format
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(bytes.NewBufferString("not json")),
Header: make(http.Header),
}
case "/constellation/v1/versions/stream/stable/major/v2/image.json": // inconsistent list
case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v2/image.json": // inconsistent list
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(bytes.NewBuffer(inconsistentListJSON)),
Header: make(http.Header),
}
case "/constellation/v1/versions/stream/stable/major/v3/image.json": // does not match requested version
case "/constellation/v1/ref/test-ref/stream/nightly/versions/major/v3/image.json": // does not match requested version
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(bytes.NewBuffer(minorListJSON)),
@ -168,10 +168,10 @@ func TestList(t *testing.T) {
})
testCases := map[string]struct {
stream, granularity, base, kind string
overrideFile string
wantList List
wantErr bool
ref, stream, granularity, base, kind string
overrideFile string
wantList List
wantErr bool
}{
"major list fetched remotely": {
wantList: *majorList(),
@ -208,7 +208,8 @@ func TestList(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
stream := "stable"
ref := "test-ref"
stream := "nightly"
granularity := "major"
base := "v1"
kind := "image"
@ -228,7 +229,7 @@ func TestList(t *testing.T) {
fetcher := &Fetcher{
httpc: client,
}
list, err := fetcher.list(context.Background(), stream, granularity, base, kind)
list, err := fetcher.list(context.Background(), ref, stream, granularity, base, kind)
if tc.wantErr {
assert.Error(err)
return
@ -256,7 +257,8 @@ func newTestClient(fn roundTripFunc) *http.Client {
func majorList() *List {
return &List{
Stream: "stable",
Ref: "test-ref",
Stream: "nightly",
Granularity: "major",
Base: "v1",
Kind: "image",
@ -268,7 +270,8 @@ func majorList() *List {
func minorList() *List {
return &List{
Stream: "stable",
Ref: "test-ref",
Stream: "nightly",
Granularity: "minor",
Base: "v1.1",
Kind: "image",
@ -278,21 +281,67 @@ func minorList() *List {
}
}
func TestIsValidStream(t *testing.T) {
func TestIsValidRef(t *testing.T) {
testCases := map[string]bool{
"stable": true,
"debug": true,
"beta": false,
"alpha": false,
"unknown": false,
"fast": false,
"feat/foo": false,
"feat-foo": true,
"feat$foo": false,
"3234": true,
"feat foo": false,
"refs-heads-feat-foo": false,
"": false,
}
for name, want := range testCases {
t.Run(name, func(t *testing.T) {
for ref, want := range testCases {
t.Run(ref, func(t *testing.T) {
assert := assert.New(t)
assert.Equal(want, IsValidStream(name))
assert.Equal(want, IsValidRef(ref))
})
}
}
func TestCanonicalRef(t *testing.T) {
testCases := map[string]string{
"feat/foo": "feat-foo",
"feat-foo": "feat-foo",
"feat$foo": "feat-foo",
"3234": "3234",
"feat foo": "feat-foo",
}
for ref, want := range testCases {
t.Run(ref, func(t *testing.T) {
assert := assert.New(t)
assert.Equal(want, CanonicalRef(ref))
})
}
}
func TestIsValidStream(t *testing.T) {
testCases := []struct {
branch string
stream string
want bool
}{
{branch: "-", stream: "stable", want: true},
{branch: "-", stream: "debug", want: true},
{branch: "-", stream: "nightly", want: false},
{branch: "-", stream: "console", want: true},
{branch: "main", stream: "stable", want: false},
{branch: "main", stream: "debug", want: true},
{branch: "main", stream: "nightly", want: true},
{branch: "main", stream: "console", want: true},
{branch: "foo-branch", stream: "nightly", want: true},
{branch: "foo-branch", stream: "console", want: true},
{branch: "foo-branch", stream: "debug", want: true},
{branch: "foo-branch", stream: "stable", want: false},
}
for _, tc := range testCases {
t.Run(tc.branch+"+"+tc.stream, func(t *testing.T) {
assert := assert.New(t)
assert.Equal(tc.want, IsValidStream(tc.branch, tc.stream))
})
}
}