/* Copyright (c) Edgeless Systems GmbH SPDX-License-Identifier: AGPL-3.0-only */ package main import ( "context" "errors" "fmt" "io" "log" "regexp" "strings" "time" compute "cloud.google.com/go/compute/apiv1" "cloud.google.com/go/compute/apiv1/computepb" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" armcomputev4 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/smithy-go" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/versionsapi" verclient "github.com/edgelesssys/constellation/v2/internal/versionsapi/client" gaxv2 "github.com/googleapis/gax-go/v2" "github.com/spf13/cobra" "go.uber.org/multierr" "go.uber.org/zap/zapcore" ) func newRemoveCmd() *cobra.Command { cmd := &cobra.Command{ Use: "remove", Short: "Remove a version/ref", Long: `Remove a version/ref from the versions API. Developers should not use this command directly. It is invoked by the CI/CD pipeline. Most developers won't have the required permissions to use this command. ❗ If you use the command nevertheless, you better know what you do. `, RunE: runRemove, Args: cobra.ExactArgs(0), } cmd.Flags().String("ref", "", "Ref to delete from.") cmd.Flags().String("stream", "", "Stream to delete from.") cmd.Flags().String("version", "", "Version to delete. The versioned objects are deleted.") cmd.Flags().String("version-path", "", "Short path of a single version to delete. The versioned objects are deleted.") cmd.Flags().Bool("all", false, "Delete the entire ref. All versions and versioned objects are deleted.") cmd.Flags().Bool("dryrun", false, "Whether to run in dry-run mode (no changes are made)") cmd.Flags().String("gcp-project", "constellation-images", "GCP project to use") cmd.Flags().String("az-subscription", "0d202bbb-4fa7-4af8-8125-58c269a05435", "Azure subscription to use") cmd.Flags().String("az-location", "northeurope", "Azure location to use") cmd.Flags().String("az-resource-group", "constellation-images", "Azure resource group to use") cmd.MarkFlagsRequiredTogether("stream", "version") cmd.MarkFlagsMutuallyExclusive("all", "stream") cmd.MarkFlagsMutuallyExclusive("all", "version") cmd.MarkFlagsMutuallyExclusive("all", "version-path") cmd.MarkFlagsMutuallyExclusive("version-path", "ref") cmd.MarkFlagsMutuallyExclusive("version-path", "stream") cmd.MarkFlagsMutuallyExclusive("version-path", "version") return cmd } func runRemove(cmd *cobra.Command, args []string) (retErr error) { flags, err := parseRmFlags(cmd) if err != nil { return err } log := logger.New(logger.PlainLog, flags.logLevel) log.Debugf("Parsed flags: %+v", flags) log.Debugf("Validating flags.") if err := flags.validate(); err != nil { return err } log.Debugf("Creating GCP client.") gcpClient, err := newGCPClient(cmd.Context(), flags.gcpProject) if err != nil { return fmt.Errorf("creating GCP client: %w", err) } log.Debugf("Creating AWS client.") awsClient, err := newAWSClient(cmd.Context(), flags.region) if err != nil { return fmt.Errorf("creating AWS client: %w", err) } log.Debugf("Creating Azure client.") azClient, err := newAzureClient(flags.azSubscription, flags.azLocation, flags.azResourceGroup) if err != nil { return fmt.Errorf("creating Azure client: %w", err) } log.Debugf("Creating versions API client.") verclient, err := verclient.NewClient(cmd.Context(), flags.region, flags.bucket, flags.distributionID, flags.dryrun, log) if err != nil { return fmt.Errorf("creating client: %w", err) } defer func(retErr *error) { log.Infof("Invalidating cache. This may take some time.") if err := verclient.InvalidateCache(cmd.Context()); err != nil && retErr == nil { *retErr = fmt.Errorf("invalidating cache: %w", err) } }(&retErr) imageClients := rmImageClients{ version: verclient, gcp: gcpClient, aws: awsClient, az: azClient, } if flags.all { log.Infof("Deleting ref %s", flags.ref) if err := deleteRef(cmd.Context(), imageClients, flags.ref, flags.dryrun, log); err != nil { return fmt.Errorf("deleting ref: %w", err) } return nil } log.Infof("Deleting single version %s", flags.ver.ShortPath()) if err := deleteSingleVersion(cmd.Context(), imageClients, flags.ver, flags.dryrun, log); err != nil { return fmt.Errorf("deleting single version: %w", err) } return nil } func deleteSingleVersion(ctx context.Context, clients rmImageClients, ver versionsapi.Version, dryrun bool, log *logger.Logger) error { var retErr error log.Debugf("Deleting version %s from versions API", ver.Version) if err := clients.version.DeleteVersion(ctx, ver); err != nil { retErr = multierr.Append(retErr, fmt.Errorf("deleting version from versions API: %w", err)) } log.Debugf("Deleting images for %s", ver.Version) if err := deleteImage(ctx, clients, ver, dryrun, log); err != nil { retErr = multierr.Append(retErr, fmt.Errorf("deleting images: %w", err)) } return retErr } func deleteRef(ctx context.Context, clients rmImageClients, ref string, dryrun bool, log *logger.Logger) error { var retErr error var vers []versionsapi.Version for _, stream := range []string{"nightly", "console", "debug"} { log.Infof("Listing versions of stream %s", stream) minorVersions, err := listMinorVersions(ctx, clients.version, ref, stream) var notFoundErr *verclient.NotFoundError if errors.As(err, ¬FoundErr) { log.Debugf("No minor versions found for stream %s", stream) continue } else if err != nil { return fmt.Errorf("listing minor versions for stream %s: %w", stream, err) } patchVersions, err := listPatchVersions(ctx, clients.version, ref, stream, minorVersions) if errors.As(err, ¬FoundErr) { log.Debugf("No patch versions found for stream %s", stream) continue } else if err != nil { return fmt.Errorf("listing patch versions for stream %s: %w", stream, err) } vers = append(vers, patchVersions...) } log.Infof("Found %d versions to delete", len(vers)) for _, ver := range vers { if err := deleteImage(ctx, clients, ver, dryrun, log); err != nil { retErr = multierr.Append(retErr, fmt.Errorf("deleting images for version %s: %w", ver.Version, err)) } } log.Infof("Deleting ref %s from versions API", ref) if err := clients.version.DeleteRef(ctx, ref); err != nil { retErr = multierr.Append(retErr, fmt.Errorf("deleting ref from versions API: %w", err)) } return retErr } func deleteImage(ctx context.Context, clients rmImageClients, ver versionsapi.Version, dryrun bool, log *logger.Logger) error { var retErr error imageInfo := versionsapi.ImageInfo{ Ref: ver.Ref, Stream: ver.Stream, Version: ver.Version, } imageInfo, err := clients.version.FetchImageInfo(ctx, imageInfo) var notFound *verclient.NotFoundError if errors.As(err, ¬Found) { log.Warnf("Image info for %s not found.", ver.Version) log.Warnf("Skipping image deletion.") return nil } else if err != nil { return fmt.Errorf("fetching image info: %w", err) } log.Infof("Deleting AWS images from %s", imageInfo.JSONPath()) for awsRegion, awsImage := range imageInfo.AWS { if err := clients.aws.deleteImage(ctx, awsImage, awsRegion, dryrun, log); err != nil { retErr = multierr.Append(retErr, fmt.Errorf("deleting AWS image %s: %w", awsImage, err)) } } log.Infof("Deleting GCP images from %s", imageInfo.JSONPath()) for _, gcpImage := range imageInfo.GCP { if err := clients.gcp.deleteImage(ctx, gcpImage, dryrun, log); err != nil { retErr = multierr.Append(retErr, fmt.Errorf("deleting GCP image %s: %w", gcpImage, err)) } } log.Infof("Deleting Azure images from %s", imageInfo.JSONPath()) for _, azImage := range imageInfo.Azure { if err := clients.az.deleteImage(ctx, azImage, dryrun, log); err != nil { retErr = multierr.Append(retErr, fmt.Errorf("deleting Azure image %s: %w", azImage, err)) } } // TODO(katexochen): Implement versions API trash. In case of failure, we should // collect the resources that couldn't be deleted and store them in the trash, so // that we can retry deleting them later. return retErr } type rmImageClients struct { version *verclient.Client gcp *gcpClient aws *awsClient az *azureClient } type rmFlags struct { ref string stream string version string versionPath string all bool dryrun bool region string bucket string distributionID string gcpProject string azSubscription string azLocation string azResourceGroup string logLevel zapcore.Level ver versionsapi.Version } func (f *rmFlags) validate() error { if f.ref == versionsapi.ReleaseRef { return fmt.Errorf("cannot delete from release ref") } if f.all { if err := versionsapi.ValidateRef(f.ref); err != nil { return fmt.Errorf("invalid ref: %w", err) } if f.ref == "main" { return fmt.Errorf("cannot delete 'main' ref") } return nil } if f.versionPath != "" { ver, err := versionsapi.NewVersionFromShortPath(f.versionPath, versionsapi.VersionKindImage) if err != nil { return fmt.Errorf("invalid version path: %w", err) } f.ver = ver return nil } ver := versionsapi.Version{ Ref: f.ref, Stream: f.stream, Version: f.version, Kind: versionsapi.VersionKindImage, } if err := ver.Validate(); err != nil { return fmt.Errorf("invalid version: %w", err) } f.ver = ver return nil } func parseRmFlags(cmd *cobra.Command) (*rmFlags, error) { ref, err := cmd.Flags().GetString("ref") if err != nil { return nil, err } ref = versionsapi.CanonicalizeRef(ref) stream, err := cmd.Flags().GetString("stream") if err != nil { return nil, err } version, err := cmd.Flags().GetString("version") if err != nil { return nil, err } versionPath, err := cmd.Flags().GetString("version-path") if err != nil { return nil, err } all, err := cmd.Flags().GetBool("all") if err != nil { return nil, err } dryrun, err := cmd.Flags().GetBool("dryrun") if err != nil { return nil, err } region, err := cmd.Flags().GetString("region") if err != nil { return nil, err } bucket, err := cmd.Flags().GetString("bucket") if err != nil { return nil, err } distributionID, err := cmd.Flags().GetString("distribution-id") if err != nil { return nil, err } gcpProject, err := cmd.Flags().GetString("gcp-project") if err != nil { return nil, err } azSubscription, err := cmd.Flags().GetString("az-subscription") if err != nil { return nil, err } azLocation, err := cmd.Flags().GetString("az-location") if err != nil { return nil, err } azResourceGroup, err := cmd.Flags().GetString("az-resource-group") if err != nil { return nil, err } verbose, err := cmd.Flags().GetBool("verbose") if err != nil { return nil, err } logLevel := zapcore.InfoLevel if verbose { logLevel = zapcore.DebugLevel } return &rmFlags{ ref: ref, stream: stream, version: version, versionPath: versionPath, all: all, dryrun: dryrun, region: region, bucket: bucket, distributionID: distributionID, gcpProject: gcpProject, azSubscription: azSubscription, azLocation: azLocation, azResourceGroup: azResourceGroup, logLevel: logLevel, }, nil } type awsClient struct { ec2 ec2API } // newAWSClient creates a new awsClient. // Requires IAM permission 'ec2:DeregisterImage'. func newAWSClient(ctx context.Context, region string) (*awsClient, error) { return &awsClient{}, nil } type ec2API interface { DeregisterImage(ctx context.Context, params *ec2.DeregisterImageInput, optFns ...func(*ec2.Options), ) (*ec2.DeregisterImageOutput, error) DescribeImages(ctx context.Context, params *ec2.DescribeImagesInput, optFns ...func(*ec2.Options), ) (*ec2.DescribeImagesOutput, error) DeleteSnapshot(ctx context.Context, params *ec2.DeleteSnapshotInput, optFns ...func(*ec2.Options), ) (*ec2.DeleteSnapshotOutput, error) } func (a *awsClient) deleteImage(ctx context.Context, ami string, region string, dryrun bool, log *logger.Logger) error { cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) if err != nil { return err } a.ec2 = ec2.NewFromConfig(cfg) snapshotID, err := a.getSnapshotID(ctx, ami, log) if err != nil { log.Warnf("Failed to get AWS snapshot ID for image %s: %v", ami, err) } if err := a.deregisterImage(ctx, ami, dryrun, log); err != nil { return fmt.Errorf("deregistering image %s: %w", ami, err) } if snapshotID != "" { if err := a.deleteSnapshot(ctx, snapshotID, dryrun, log); err != nil { return fmt.Errorf("deleting snapshot %s: %w", snapshotID, err) } } return nil } func (a *awsClient) deregisterImage(ctx context.Context, ami string, dryrun bool, log *logger.Logger) error { log.Debugf("Deregistering image %s", ami) deregisterReq := ec2.DeregisterImageInput{ ImageId: &ami, DryRun: &dryrun, } _, err := a.ec2.DeregisterImage(ctx, &deregisterReq) var apiErr smithy.APIError if errors.As(err, &apiErr) && (apiErr.ErrorCode() == "InvalidAMIID.NotFound" || apiErr.ErrorCode() == "InvalidAMIID.Unavailable") { log.Warnf("AWS image %s not found", ami) return nil } return err } func (a *awsClient) getSnapshotID(ctx context.Context, ami string, log *logger.Logger) (string, error) { log.Debugf("Describing image %s", ami) req := ec2.DescribeImagesInput{ ImageIds: []string{ami}, } resp, err := a.ec2.DescribeImages(ctx, &req) if err != nil { return "", fmt.Errorf("describing image %s: %w", ami, err) } if len(resp.Images) == 0 { return "", fmt.Errorf("image %s not found", ami) } if len(resp.Images) > 1 { return "", fmt.Errorf("found multiple images with ami %s", ami) } image := resp.Images[0] if len(image.BlockDeviceMappings) != 1 { return "", fmt.Errorf("found %d block device mappings for image %s, expected 1", len(image.BlockDeviceMappings), ami) } if image.BlockDeviceMappings[0].Ebs == nil { return "", fmt.Errorf("image %s does not have an EBS block device mapping", ami) } ebs := image.BlockDeviceMappings[0].Ebs if ebs.SnapshotId == nil { return "", fmt.Errorf("image %s does not have an EBS snapshot", ami) } snapshotID := *ebs.SnapshotId return snapshotID, nil } func (a *awsClient) deleteSnapshot(ctx context.Context, snapshotID string, dryrun bool, log *logger.Logger) error { log.Debugf("Deleting AWS snapshot %s", snapshotID) req := ec2.DeleteSnapshotInput{ SnapshotId: &snapshotID, DryRun: &dryrun, } _, err := a.ec2.DeleteSnapshot(ctx, &req) var apiErr smithy.APIError if errors.As(err, &apiErr) && (apiErr.ErrorCode() == "InvalidSnapshot.NotFound" || apiErr.ErrorCode() == "InvalidSnapshot.Unavailable") { log.Warnf("AWS snapshot %s not found", snapshotID) return nil } return err } type gcpClient struct { project string compute gcpComputeAPI } func newGCPClient(ctx context.Context, project string) (*gcpClient, error) { compute, err := compute.NewImagesRESTClient(ctx) if err != nil { return nil, err } return &gcpClient{ compute: compute, project: project, }, nil } type gcpComputeAPI interface { Delete(ctx context.Context, req *computepb.DeleteImageRequest, opts ...gaxv2.CallOption, ) (*compute.Operation, error) io.Closer } func (g *gcpClient) deleteImage(ctx context.Context, image string, dryrun bool, log *logger.Logger) error { req := &computepb.DeleteImageRequest{ Image: image, Project: g.project, } if dryrun { log.Debugf("DryRun: delete image request: %v", req) return nil } log.Debugf("Deleting image %s", image) op, err := g.compute.Delete(ctx, req) if err != nil && strings.Contains(err.Error(), "404") { log.Warnf("GCP image %s not found", image) return nil } else if err != nil { return fmt.Errorf("deleting image %s: %w", image, err) } log.Debugf("Waiting for operation to finish.") if err := op.Wait(ctx); err != nil { return fmt.Errorf("waiting for operation: %w", err) } return nil } func (g *gcpClient) Close() error { return g.compute.Close() } type azureClient struct { subscription string location string resourceGroup string galleries azureGalleriesAPI image azureGalleriesImageAPI imageVersions azureGalleriesImageVersionAPI } func newAzureClient(subscription, location, resourceGroup string) (*azureClient, error) { cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { log.Fatal(err) } galleriesClient, err := armcomputev4.NewGalleriesClient(subscription, cred, nil) if err != nil { return nil, err } galleriesImageClient, err := armcomputev4.NewGalleryImagesClient(subscription, cred, nil) if err != nil { return nil, err } galleriesImageVersionClient, err := armcomputev4.NewGalleryImageVersionsClient(subscription, cred, nil) if err != nil { return nil, err } return &azureClient{ subscription: subscription, location: location, resourceGroup: resourceGroup, galleries: galleriesClient, image: galleriesImageClient, imageVersions: galleriesImageVersionClient, }, nil } type azureGalleriesAPI interface { NewListPager(options *armcomputev4.GalleriesClientListOptions, ) *runtime.Pager[armcomputev4.GalleriesClientListResponse] } type azureGalleriesImageAPI interface { BeginDelete(ctx context.Context, resourceGroupName string, galleryName string, galleryImageName string, options *armcomputev4.GalleryImagesClientBeginDeleteOptions, ) (*runtime.Poller[armcomputev4.GalleryImagesClientDeleteResponse], error) } type azureGalleriesImageVersionAPI interface { NewListByGalleryImagePager(resourceGroupName string, galleryName string, galleryImageName string, options *armcomputev4.GalleryImageVersionsClientListByGalleryImageOptions, ) *runtime.Pager[armcomputev4.GalleryImageVersionsClientListByGalleryImageResponse] BeginDelete(ctx context.Context, resourceGroupName string, galleryName string, galleryImageName string, galleryImageVersionName string, options *armcomputev4.GalleryImageVersionsClientBeginDeleteOptions, ) (*runtime.Poller[armcomputev4.GalleryImageVersionsClientDeleteResponse], error) } var ( azImageRegex = regexp.MustCompile("^/subscriptions/[[:alnum:]._-]+/resourceGroups/([[:alnum:]._-]+)/providers/Microsoft.Compute/galleries/([[:alnum:]._-]+)/images/([[:alnum:]._-]+)/versions/([[:alnum:]._-]+)$") azCommunityImageRegex = regexp.MustCompile("^/CommunityGalleries/([[:alnum:]-]+)/Images/([[:alnum:]._-]+)/Versions/([[:alnum:]._-]+)$") ) func (a *azureClient) deleteImage(ctx context.Context, image string, dryrun bool, log *logger.Logger) error { azImage, err := a.parseImage(ctx, image, log) if err != nil { return err } if dryrun { log.Debugf("DryRun: delete image %v", azImage) return nil } log.Debugf("Deleting image %q, version %q", azImage.imageDefinition, azImage.version) poller, err := a.imageVersions.BeginDelete(ctx, azImage.resourceGroup, azImage.gallery, azImage.imageDefinition, azImage.version, nil) if err != nil { return fmt.Errorf("begin delete image version: %w", err) } log.Debugf("Waiting for operation to finish.") if _, err := poller.PollUntilDone(ctx, nil); err != nil { return fmt.Errorf("waiting for operation: %w", err) } log.Debugf("Checking if image definition %q still has versions left", azImage.imageDefinition) pager := a.imageVersions.NewListByGalleryImagePager(azImage.resourceGroup, azImage.gallery, azImage.imageDefinition, nil) for pager.More() { nextResult, err := pager.NextPage(ctx) if err != nil { return fmt.Errorf("listing image versions of image definition %s: %w", azImage.imageDefinition, err) } if len(nextResult.Value) != 0 { log.Debugf("Image definition %q still has versions left, won't be deleted", azImage.imageDefinition) return nil } } time.Sleep(15 * time.Second) // Azure needs time understand that there is no version left... log.Debugf("Deleting image definition %s", azImage.imageDefinition) op, err := a.image.BeginDelete(ctx, azImage.resourceGroup, azImage.gallery, azImage.imageDefinition, nil) if err != nil { return fmt.Errorf("deleting image definition %s: %w", azImage.imageDefinition, err) } log.Debugf("Waiting for operation to finish.") if _, err := op.PollUntilDone(ctx, nil); err != nil { return fmt.Errorf("waiting for operation: %w", err) } return nil } type azImage struct { resourceGroup string gallery string imageDefinition string version string } func (a *azureClient) parseImage(ctx context.Context, image string, log *logger.Logger) (azImage, error) { if m := azImageRegex.FindStringSubmatch(image); len(m) == 5 { log.Debugf( "Image matches local image format, resource group: %s, gallery: %s, image definition: %s, version: %s.", m[1], m[2], m[3], m[4], ) return azImage{ resourceGroup: m[1], gallery: m[2], imageDefinition: m[3], version: m[4], }, nil } if !azCommunityImageRegex.MatchString(image) { return azImage{}, fmt.Errorf("invalid image %s", image) } m := azCommunityImageRegex.FindStringSubmatch(image) galleryPublicName := m[1] imageDefinition := m[2] version := m[3] log.Debugf( "Image matches community image format, gallery public name: %s, image definition: %s, version: %s.", galleryPublicName, imageDefinition, version, ) var galleryName string pager := a.galleries.NewListPager(nil) for pager.More() { nextResult, err := pager.NextPage(ctx) if err != nil { return azImage{}, fmt.Errorf("failed to advance page: %w", err) } for _, v := range nextResult.Value { if v.Name == nil { log.Debugf("Skipping gallery with nil name.") continue } if v.Properties.SharingProfile == nil { log.Debugf("Skipping gallery %s with nil sharing profile.", *v.Name) continue } if v.Properties.SharingProfile.CommunityGalleryInfo == nil { log.Debugf("Skipping gallery %s with nil community gallery info.", *v.Name) continue } if v.Properties.SharingProfile.CommunityGalleryInfo.PublicNames == nil { log.Debugf("Skipping gallery %s with nil public names.", *v.Name) continue } for _, publicName := range v.Properties.SharingProfile.CommunityGalleryInfo.PublicNames { if publicName == nil { log.Debugf("Skipping nil public name.") continue } if *publicName == galleryPublicName { galleryName = *v.Name break } } if galleryName != "" { break } } } if galleryName == "" { return azImage{}, fmt.Errorf("failed to find gallery for public name %s", galleryPublicName) } return azImage{ resourceGroup: a.resourceGroup, gallery: galleryName, imageDefinition: imageDefinition, version: version, }, nil }