constellation/internal/versionsapi/cli/rm.go

768 lines
23 KiB
Go
Raw Normal View History

/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package main
import (
"context"
"errors"
"fmt"
"io"
"log"
"regexp"
"strings"
"time"
compute "cloud.google.com/go/compute/apiv1"
"cloud.google.com/go/compute/apiv1/computepb"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
armcomputev4 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/smithy-go"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/versionsapi"
verclient "github.com/edgelesssys/constellation/v2/internal/versionsapi/client"
gaxv2 "github.com/googleapis/gax-go/v2"
"github.com/spf13/cobra"
"go.uber.org/multierr"
"go.uber.org/zap/zapcore"
)
func newRemoveCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "remove",
Short: "Remove a version/ref",
Long: `Remove a version/ref from the versions API.
Developers should not use this command directly. It is invoked by the CI/CD pipeline.
Most developers won't have the required permissions to use this command.
If you use the command nevertheless, you better know what you do.
`,
RunE: runRemove,
Args: cobra.ExactArgs(0),
}
cmd.Flags().String("ref", "", "Ref to delete from.")
cmd.Flags().String("stream", "", "Stream to delete from.")
cmd.Flags().String("version", "", "Version to delete. The versioned objects are deleted.")
cmd.Flags().String("version-path", "", "Short path of a single version to delete. The versioned objects are deleted.")
cmd.Flags().Bool("all", false, "Delete the entire ref. All versions and versioned objects are deleted.")
cmd.Flags().Bool("dryrun", false, "Whether to run in dry-run mode (no changes are made)")
cmd.Flags().String("gcp-project", "constellation-images", "GCP project to use")
cmd.Flags().String("az-subscription", "0d202bbb-4fa7-4af8-8125-58c269a05435", "Azure subscription to use")
cmd.Flags().String("az-location", "northeurope", "Azure location to use")
cmd.Flags().String("az-resource-group", "constellation-images", "Azure resource group to use")
cmd.MarkFlagsRequiredTogether("stream", "version")
cmd.MarkFlagsMutuallyExclusive("all", "stream")
cmd.MarkFlagsMutuallyExclusive("all", "version")
cmd.MarkFlagsMutuallyExclusive("all", "version-path")
cmd.MarkFlagsMutuallyExclusive("version-path", "ref")
cmd.MarkFlagsMutuallyExclusive("version-path", "stream")
cmd.MarkFlagsMutuallyExclusive("version-path", "version")
return cmd
}
func runRemove(cmd *cobra.Command, args []string) (retErr error) {
flags, err := parseRmFlags(cmd)
if err != nil {
return err
}
log := logger.New(logger.PlainLog, flags.logLevel)
log.Debugf("Parsed flags: %+v", flags)
log.Debugf("Validating flags.")
if err := flags.validate(); err != nil {
return err
}
log.Debugf("Creating GCP client.")
gcpClient, err := newGCPClient(cmd.Context(), flags.gcpProject)
if err != nil {
return fmt.Errorf("creating GCP client: %w", err)
}
log.Debugf("Creating AWS client.")
awsClient, err := newAWSClient(cmd.Context(), flags.region)
if err != nil {
return fmt.Errorf("creating AWS client: %w", err)
}
log.Debugf("Creating Azure client.")
azClient, err := newAzureClient(flags.azSubscription, flags.azLocation, flags.azResourceGroup)
if err != nil {
return fmt.Errorf("creating Azure client: %w", err)
}
log.Debugf("Creating versions API client.")
verclient, err := verclient.NewClient(cmd.Context(), flags.region, flags.bucket, flags.distributionID, flags.dryrun, log)
if err != nil {
return fmt.Errorf("creating client: %w", err)
}
defer func(retErr *error) {
log.Infof("Invalidating cache. This may take some time.")
if err := verclient.InvalidateCache(cmd.Context()); err != nil && retErr == nil {
*retErr = fmt.Errorf("invalidating cache: %w", err)
}
}(&retErr)
imageClients := rmImageClients{
version: verclient,
gcp: gcpClient,
aws: awsClient,
az: azClient,
}
if flags.all {
log.Infof("Deleting ref %s", flags.ref)
if err := deleteRef(cmd.Context(), imageClients, flags.ref, flags.dryrun, log); err != nil {
return fmt.Errorf("deleting ref: %w", err)
}
return nil
}
log.Infof("Deleting single version %s", flags.ver.ShortPath())
if err := deleteSingleVersion(cmd.Context(), imageClients, flags.ver, flags.dryrun, log); err != nil {
return fmt.Errorf("deleting single version: %w", err)
}
return nil
}
func deleteSingleVersion(ctx context.Context, clients rmImageClients, ver versionsapi.Version, dryrun bool, log *logger.Logger) error {
var retErr error
log.Debugf("Deleting images for %s", ver.Version)
if err := deleteImage(ctx, clients, ver, dryrun, log); err != nil {
retErr = multierr.Append(retErr, fmt.Errorf("deleting images: %w", err))
}
log.Debugf("Deleting version %s from versions API", ver.Version)
if err := clients.version.DeleteVersion(ctx, ver); err != nil {
retErr = multierr.Append(retErr, fmt.Errorf("deleting version from versions API: %w", err))
}
return retErr
}
func deleteRef(ctx context.Context, clients rmImageClients, ref string, dryrun bool, log *logger.Logger) error {
var retErr error
var vers []versionsapi.Version
for _, stream := range []string{"nightly", "console", "debug"} {
log.Infof("Listing versions of stream %s", stream)
minorVersions, err := listMinorVersions(ctx, clients.version, ref, stream)
var notFoundErr *verclient.NotFoundError
if errors.As(err, &notFoundErr) {
log.Debugf("No minor versions found for stream %s", stream)
continue
} else if err != nil {
return fmt.Errorf("listing minor versions for stream %s: %w", stream, err)
}
patchVersions, err := listPatchVersions(ctx, clients.version, ref, stream, minorVersions)
if errors.As(err, &notFoundErr) {
log.Debugf("No patch versions found for stream %s", stream)
continue
} else if err != nil {
return fmt.Errorf("listing patch versions for stream %s: %w", stream, err)
}
vers = append(vers, patchVersions...)
}
log.Infof("Found %d versions to delete", len(vers))
for _, ver := range vers {
if err := deleteImage(ctx, clients, ver, dryrun, log); err != nil {
retErr = multierr.Append(retErr, fmt.Errorf("deleting images for version %s: %w", ver.Version, err))
}
}
log.Infof("Deleting ref %s from versions API", ref)
if err := clients.version.DeleteRef(ctx, ref); err != nil {
retErr = multierr.Append(retErr, fmt.Errorf("deleting ref from versions API: %w", err))
}
return retErr
}
func deleteImage(ctx context.Context, clients rmImageClients, ver versionsapi.Version, dryrun bool, log *logger.Logger) error {
var retErr error
imageInfo := versionsapi.ImageInfo{
Ref: ver.Ref,
Stream: ver.Stream,
Version: ver.Version,
}
imageInfo, err := clients.version.FetchImageInfo(ctx, imageInfo)
var notFound *verclient.NotFoundError
if errors.As(err, &notFound) {
log.Warnf("Image info for %s not found.", ver.Version)
log.Warnf("Skipping image deletion.")
return nil
} else if err != nil {
return fmt.Errorf("fetching image info: %w", err)
}
log.Infof("Deleting AWS images from %s", imageInfo.JSONPath())
for awsRegion, awsImage := range imageInfo.AWS {
if err := clients.aws.deleteImage(ctx, awsImage, awsRegion, dryrun, log); err != nil {
retErr = multierr.Append(retErr, fmt.Errorf("deleting AWS image %s: %w", awsImage, err))
}
}
log.Infof("Deleting GCP images from %s", imageInfo.JSONPath())
for _, gcpImage := range imageInfo.GCP {
if err := clients.gcp.deleteImage(ctx, gcpImage, dryrun, log); err != nil {
retErr = multierr.Append(retErr, fmt.Errorf("deleting GCP image %s: %w", gcpImage, err))
}
}
log.Infof("Deleting Azure images from %s", imageInfo.JSONPath())
for _, azImage := range imageInfo.Azure {
if err := clients.az.deleteImage(ctx, azImage, dryrun, log); err != nil {
retErr = multierr.Append(retErr, fmt.Errorf("deleting Azure image %s: %w", azImage, err))
}
}
// TODO(katexochen): Implement versions API trash. In case of failure, we should
// collect the resources that couldn't be deleted and store them in the trash, so
// that we can retry deleting them later.
return retErr
}
type rmImageClients struct {
version *verclient.Client
gcp *gcpClient
aws *awsClient
az *azureClient
}
type rmFlags struct {
ref string
stream string
version string
versionPath string
all bool
dryrun bool
region string
bucket string
distributionID string
gcpProject string
azSubscription string
azLocation string
azResourceGroup string
logLevel zapcore.Level
ver versionsapi.Version
}
func (f *rmFlags) validate() error {
if f.ref == versionsapi.ReleaseRef {
return fmt.Errorf("cannot delete from release ref")
}
if f.all {
if err := versionsapi.ValidateRef(f.ref); err != nil {
return fmt.Errorf("invalid ref: %w", err)
}
if f.ref == "main" {
return fmt.Errorf("cannot delete 'main' ref")
}
return nil
}
if f.versionPath != "" {
ver, err := versionsapi.NewVersionFromShortPath(f.versionPath, versionsapi.VersionKindImage)
if err != nil {
return fmt.Errorf("invalid version path: %w", err)
}
f.ver = ver
return nil
}
ver := versionsapi.Version{
Ref: f.ref,
Stream: f.stream,
Version: f.version,
Kind: versionsapi.VersionKindImage,
}
if err := ver.Validate(); err != nil {
return fmt.Errorf("invalid version: %w", err)
}
f.ver = ver
return nil
}
func parseRmFlags(cmd *cobra.Command) (*rmFlags, error) {
ref, err := cmd.Flags().GetString("ref")
if err != nil {
return nil, err
}
ref = versionsapi.CanonicalizeRef(ref)
stream, err := cmd.Flags().GetString("stream")
if err != nil {
return nil, err
}
version, err := cmd.Flags().GetString("version")
if err != nil {
return nil, err
}
versionPath, err := cmd.Flags().GetString("version-path")
if err != nil {
return nil, err
}
all, err := cmd.Flags().GetBool("all")
if err != nil {
return nil, err
}
dryrun, err := cmd.Flags().GetBool("dryrun")
if err != nil {
return nil, err
}
region, err := cmd.Flags().GetString("region")
if err != nil {
return nil, err
}
bucket, err := cmd.Flags().GetString("bucket")
if err != nil {
return nil, err
}
distributionID, err := cmd.Flags().GetString("distribution-id")
if err != nil {
return nil, err
}
gcpProject, err := cmd.Flags().GetString("gcp-project")
if err != nil {
return nil, err
}
azSubscription, err := cmd.Flags().GetString("az-subscription")
if err != nil {
return nil, err
}
azLocation, err := cmd.Flags().GetString("az-location")
if err != nil {
return nil, err
}
azResourceGroup, err := cmd.Flags().GetString("az-resource-group")
if err != nil {
return nil, err
}
verbose, err := cmd.Flags().GetBool("verbose")
if err != nil {
return nil, err
}
logLevel := zapcore.InfoLevel
if verbose {
logLevel = zapcore.DebugLevel
}
return &rmFlags{
ref: ref,
stream: stream,
version: version,
versionPath: versionPath,
all: all,
dryrun: dryrun,
region: region,
bucket: bucket,
distributionID: distributionID,
gcpProject: gcpProject,
azSubscription: azSubscription,
azLocation: azLocation,
azResourceGroup: azResourceGroup,
logLevel: logLevel,
}, nil
}
type awsClient struct {
ec2 ec2API
}
// newAWSClient creates a new awsClient.
// Requires IAM permission 'ec2:DeregisterImage'.
func newAWSClient(ctx context.Context, region string) (*awsClient, error) {
return &awsClient{}, nil
}
type ec2API interface {
DeregisterImage(ctx context.Context, params *ec2.DeregisterImageInput, optFns ...func(*ec2.Options),
) (*ec2.DeregisterImageOutput, error)
DescribeImages(ctx context.Context, params *ec2.DescribeImagesInput, optFns ...func(*ec2.Options),
) (*ec2.DescribeImagesOutput, error)
DeleteSnapshot(ctx context.Context, params *ec2.DeleteSnapshotInput, optFns ...func(*ec2.Options),
) (*ec2.DeleteSnapshotOutput, error)
}
func (a *awsClient) deleteImage(ctx context.Context, ami string, region string, dryrun bool, log *logger.Logger) error {
cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region))
if err != nil {
return err
}
a.ec2 = ec2.NewFromConfig(cfg)
log.Debugf("Deleting resources in AWS region %s", region)
snapshotID, err := a.getSnapshotID(ctx, ami, log)
if err != nil {
log.Warnf("Failed to get AWS snapshot ID for image %s: %v", ami, err)
}
if err := a.deregisterImage(ctx, ami, dryrun, log); err != nil {
return fmt.Errorf("deregistering image %s: %w", ami, err)
}
if snapshotID != "" {
if err := a.deleteSnapshot(ctx, snapshotID, dryrun, log); err != nil {
return fmt.Errorf("deleting snapshot %s: %w", snapshotID, err)
}
}
return nil
}
func (a *awsClient) deregisterImage(ctx context.Context, ami string, dryrun bool, log *logger.Logger) error {
log.Debugf("Deregistering image %s", ami)
deregisterReq := ec2.DeregisterImageInput{
ImageId: &ami,
DryRun: &dryrun,
}
_, err := a.ec2.DeregisterImage(ctx, &deregisterReq)
var apiErr smithy.APIError
if errors.As(err, &apiErr) &&
(apiErr.ErrorCode() == "InvalidAMIID.NotFound" ||
apiErr.ErrorCode() == "InvalidAMIID.Unavailable") {
log.Warnf("AWS image %s not found", ami)
return nil
}
return err
}
func (a *awsClient) getSnapshotID(ctx context.Context, ami string, log *logger.Logger) (string, error) {
log.Debugf("Describing image %s", ami)
req := ec2.DescribeImagesInput{
ImageIds: []string{ami},
}
resp, err := a.ec2.DescribeImages(ctx, &req)
if err != nil {
return "", fmt.Errorf("describing image %s: %w", ami, err)
}
if len(resp.Images) == 0 {
return "", fmt.Errorf("image %s not found", ami)
}
if len(resp.Images) > 1 {
return "", fmt.Errorf("found multiple images with ami %s", ami)
}
image := resp.Images[0]
if len(image.BlockDeviceMappings) != 1 {
return "", fmt.Errorf("found %d block device mappings for image %s, expected 1", len(image.BlockDeviceMappings), ami)
}
if image.BlockDeviceMappings[0].Ebs == nil {
return "", fmt.Errorf("image %s does not have an EBS block device mapping", ami)
}
ebs := image.BlockDeviceMappings[0].Ebs
if ebs.SnapshotId == nil {
return "", fmt.Errorf("image %s does not have an EBS snapshot", ami)
}
snapshotID := *ebs.SnapshotId
return snapshotID, nil
}
func (a *awsClient) deleteSnapshot(ctx context.Context, snapshotID string, dryrun bool, log *logger.Logger) error {
log.Debugf("Deleting AWS snapshot %s", snapshotID)
req := ec2.DeleteSnapshotInput{
SnapshotId: &snapshotID,
DryRun: &dryrun,
}
_, err := a.ec2.DeleteSnapshot(ctx, &req)
var apiErr smithy.APIError
if errors.As(err, &apiErr) &&
(apiErr.ErrorCode() == "InvalidSnapshot.NotFound" ||
apiErr.ErrorCode() == "InvalidSnapshot.Unavailable") {
log.Warnf("AWS snapshot %s not found", snapshotID)
return nil
}
return err
}
type gcpClient struct {
project string
compute gcpComputeAPI
}
func newGCPClient(ctx context.Context, project string) (*gcpClient, error) {
compute, err := compute.NewImagesRESTClient(ctx)
if err != nil {
return nil, err
}
return &gcpClient{
compute: compute,
project: project,
}, nil
}
type gcpComputeAPI interface {
Delete(ctx context.Context, req *computepb.DeleteImageRequest, opts ...gaxv2.CallOption,
) (*compute.Operation, error)
io.Closer
}
func (g *gcpClient) deleteImage(ctx context.Context, image string, dryrun bool, log *logger.Logger) error {
req := &computepb.DeleteImageRequest{
Image: image,
Project: g.project,
}
if dryrun {
log.Debugf("DryRun: delete image request: %v", req)
return nil
}
log.Debugf("Deleting image %s", image)
op, err := g.compute.Delete(ctx, req)
if err != nil && strings.Contains(err.Error(), "404") {
log.Warnf("GCP image %s not found", image)
return nil
} else if err != nil {
return fmt.Errorf("deleting image %s: %w", image, err)
}
log.Debugf("Waiting for operation to finish.")
if err := op.Wait(ctx); err != nil {
return fmt.Errorf("waiting for operation: %w", err)
}
return nil
}
func (g *gcpClient) Close() error {
return g.compute.Close()
}
type azureClient struct {
subscription string
location string
resourceGroup string
galleries azureGalleriesAPI
image azureGalleriesImageAPI
imageVersions azureGalleriesImageVersionAPI
}
func newAzureClient(subscription, location, resourceGroup string) (*azureClient, error) {
cred, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
log.Fatal(err)
}
galleriesClient, err := armcomputev4.NewGalleriesClient(subscription, cred, nil)
if err != nil {
return nil, err
}
galleriesImageClient, err := armcomputev4.NewGalleryImagesClient(subscription, cred, nil)
if err != nil {
return nil, err
}
galleriesImageVersionClient, err := armcomputev4.NewGalleryImageVersionsClient(subscription, cred, nil)
if err != nil {
return nil, err
}
return &azureClient{
subscription: subscription,
location: location,
resourceGroup: resourceGroup,
galleries: galleriesClient,
image: galleriesImageClient,
imageVersions: galleriesImageVersionClient,
}, nil
}
type azureGalleriesAPI interface {
NewListPager(options *armcomputev4.GalleriesClientListOptions,
) *runtime.Pager[armcomputev4.GalleriesClientListResponse]
}
type azureGalleriesImageAPI interface {
BeginDelete(ctx context.Context, resourceGroupName string, galleryName string, galleryImageName string,
options *armcomputev4.GalleryImagesClientBeginDeleteOptions,
) (*runtime.Poller[armcomputev4.GalleryImagesClientDeleteResponse], error)
}
type azureGalleriesImageVersionAPI interface {
NewListByGalleryImagePager(resourceGroupName string, galleryName string, galleryImageName string,
options *armcomputev4.GalleryImageVersionsClientListByGalleryImageOptions,
) *runtime.Pager[armcomputev4.GalleryImageVersionsClientListByGalleryImageResponse]
BeginDelete(ctx context.Context, resourceGroupName string, galleryName string, galleryImageName string,
galleryImageVersionName string, options *armcomputev4.GalleryImageVersionsClientBeginDeleteOptions,
) (*runtime.Poller[armcomputev4.GalleryImageVersionsClientDeleteResponse], error)
}
var (
azImageRegex = regexp.MustCompile("^/subscriptions/[[:alnum:]._-]+/resourceGroups/([[:alnum:]._-]+)/providers/Microsoft.Compute/galleries/([[:alnum:]._-]+)/images/([[:alnum:]._-]+)/versions/([[:alnum:]._-]+)$")
azCommunityImageRegex = regexp.MustCompile("^/CommunityGalleries/([[:alnum:]-]+)/Images/([[:alnum:]._-]+)/Versions/([[:alnum:]._-]+)$")
)
func (a *azureClient) deleteImage(ctx context.Context, image string, dryrun bool, log *logger.Logger) error {
azImage, err := a.parseImage(ctx, image, log)
if err != nil {
return err
}
if dryrun {
log.Debugf("DryRun: delete image %v", azImage)
return nil
}
log.Debugf("Deleting image %q, version %q", azImage.imageDefinition, azImage.version)
poller, err := a.imageVersions.BeginDelete(ctx, azImage.resourceGroup, azImage.gallery,
azImage.imageDefinition, azImage.version, nil)
if err != nil {
return fmt.Errorf("begin delete image version: %w", err)
}
log.Debugf("Waiting for operation to finish.")
if _, err := poller.PollUntilDone(ctx, nil); err != nil {
return fmt.Errorf("waiting for operation: %w", err)
}
log.Debugf("Checking if image definition %q still has versions left", azImage.imageDefinition)
pager := a.imageVersions.NewListByGalleryImagePager(azImage.resourceGroup, azImage.gallery,
azImage.imageDefinition, nil)
for pager.More() {
nextResult, err := pager.NextPage(ctx)
if err != nil {
return fmt.Errorf("listing image versions of image definition %s: %w", azImage.imageDefinition, err)
}
if len(nextResult.Value) != 0 {
log.Debugf("Image definition %q still has versions left, won't be deleted", azImage.imageDefinition)
return nil
}
}
time.Sleep(15 * time.Second) // Azure needs time understand that there is no version left...
log.Debugf("Deleting image definition %s", azImage.imageDefinition)
op, err := a.image.BeginDelete(ctx, azImage.resourceGroup, azImage.gallery, azImage.imageDefinition, nil)
if err != nil {
return fmt.Errorf("deleting image definition %s: %w", azImage.imageDefinition, err)
}
log.Debugf("Waiting for operation to finish.")
if _, err := op.PollUntilDone(ctx, nil); err != nil {
return fmt.Errorf("waiting for operation: %w", err)
}
return nil
}
type azImage struct {
resourceGroup string
gallery string
imageDefinition string
version string
}
func (a *azureClient) parseImage(ctx context.Context, image string, log *logger.Logger) (azImage, error) {
if m := azImageRegex.FindStringSubmatch(image); len(m) == 5 {
log.Debugf(
"Image matches local image format, resource group: %s, gallery: %s, image definition: %s, version: %s.",
m[1], m[2], m[3], m[4],
)
return azImage{
resourceGroup: m[1],
gallery: m[2],
imageDefinition: m[3],
version: m[4],
}, nil
}
if !azCommunityImageRegex.MatchString(image) {
return azImage{}, fmt.Errorf("invalid image %s", image)
}
m := azCommunityImageRegex.FindStringSubmatch(image)
galleryPublicName := m[1]
imageDefinition := m[2]
version := m[3]
log.Debugf(
"Image matches community image format, gallery public name: %s, image definition: %s, version: %s.",
galleryPublicName, imageDefinition, version,
)
var galleryName string
pager := a.galleries.NewListPager(nil)
for pager.More() {
nextResult, err := pager.NextPage(ctx)
if err != nil {
return azImage{}, fmt.Errorf("failed to advance page: %w", err)
}
for _, v := range nextResult.Value {
if v.Name == nil {
log.Debugf("Skipping gallery with nil name.")
continue
}
if v.Properties.SharingProfile == nil {
log.Debugf("Skipping gallery %s with nil sharing profile.", *v.Name)
continue
}
if v.Properties.SharingProfile.CommunityGalleryInfo == nil {
log.Debugf("Skipping gallery %s with nil community gallery info.", *v.Name)
continue
}
if v.Properties.SharingProfile.CommunityGalleryInfo.PublicNames == nil {
log.Debugf("Skipping gallery %s with nil public names.", *v.Name)
continue
}
for _, publicName := range v.Properties.SharingProfile.CommunityGalleryInfo.PublicNames {
if publicName == nil {
log.Debugf("Skipping nil public name.")
continue
}
if *publicName == galleryPublicName {
galleryName = *v.Name
break
}
}
if galleryName != "" {
break
}
}
}
if galleryName == "" {
return azImage{}, fmt.Errorf("failed to find gallery for public name %s", galleryPublicName)
}
return azImage{
resourceGroup: a.resourceGroup,
gallery: galleryName,
imageDefinition: imageDefinition,
version: version,
}, nil
}