From 5afb80c5884847e5613e6ebd18796ed5cbce3271 Mon Sep 17 00:00:00 2001 From: Malte Poll Date: Mon, 5 Jun 2023 16:29:59 +0200 Subject: [PATCH] operators: cleanup placeholder nodeversion --- cli/internal/cmd/upgradeapply.go | 2 +- .../internal/cloud/fake/client/BUILD.bazel | 5 +- .../internal/cloud/fake/client/client.go | 5 +- .../internal/constants/constants.go | 6 ++ .../internal/deploy/BUILD.bazel | 1 + .../internal/deploy/deploy.go | 84 +++++++++++++++++++ 6 files changed, 99 insertions(+), 4 deletions(-) diff --git a/cli/internal/cmd/upgradeapply.go b/cli/internal/cmd/upgradeapply.go index 2dc57c473..c50300cac 100644 --- a/cli/internal/cmd/upgradeapply.go +++ b/cli/internal/cmd/upgradeapply.go @@ -45,7 +45,7 @@ func newUpgradeApplyCmd() *cobra.Command { cmd.Flags().BoolP("yes", "y", false, "run upgrades without further confirmation\n"+ "WARNING: might delete your resources in case you are using cert-manager in your cluster. Please read the docs.\n"+ "WARNING: might unintentionally overwrite measurements in the running cluster.") - cmd.Flags().Duration("timeout", 3*time.Minute, "change helm upgrade timeout\n"+ + cmd.Flags().Duration("timeout", 5*time.Minute, "change helm upgrade timeout\n"+ "Might be useful for slow connections or big clusters.") if err := cmd.Flags().MarkHidden("timeout"); err != nil { panic(err) diff --git a/operators/constellation-node-operator/internal/cloud/fake/client/BUILD.bazel b/operators/constellation-node-operator/internal/cloud/fake/client/BUILD.bazel index 15288c680..90a2e4d63 100644 --- a/operators/constellation-node-operator/internal/cloud/fake/client/BUILD.bazel +++ b/operators/constellation-node-operator/internal/cloud/fake/client/BUILD.bazel @@ -5,5 +5,8 @@ go_library( srcs = ["client.go"], importpath = "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/fake/client", visibility = ["//operators/constellation-node-operator:__subpackages__"], - deps = ["//operators/constellation-node-operator/api/v1alpha1"], + deps = [ + "//operators/constellation-node-operator/api/v1alpha1", + "//operators/constellation-node-operator/internal/constants", + ], ) diff --git a/operators/constellation-node-operator/internal/cloud/fake/client/client.go b/operators/constellation-node-operator/internal/cloud/fake/client/client.go index 3f256c1cb..3d3765c2a 100644 --- a/operators/constellation-node-operator/internal/cloud/fake/client/client.go +++ b/operators/constellation-node-operator/internal/cloud/fake/client/client.go @@ -11,6 +11,7 @@ import ( "fmt" updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/constants" ) const ( @@ -53,7 +54,7 @@ func (c *Client) SetScalingGroupImage(_ context.Context, _, _ string) error { // GetScalingGroupImage retrieves the image currently used by a scaling group. func (c *Client) GetScalingGroupImage(_ context.Context, _ string) (string, error) { - return "unsupportedCSP", nil + return constants.PlaceholderImageName, nil } // GetScalingGroupName retrieves the name of a scaling group. @@ -87,5 +88,5 @@ func (c *Client) ListScalingGroups(_ context.Context, _ string) (controlPlaneGro // AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler. func (c *Client) AutoscalingCloudProvider() string { - return "unsupportedCSP" + return constants.PlaceholderImageName } diff --git a/operators/constellation-node-operator/internal/constants/constants.go b/operators/constellation-node-operator/internal/constants/constants.go index 0c20e34a8..b057df59c 100644 --- a/operators/constellation-node-operator/internal/constants/constants.go +++ b/operators/constellation-node-operator/internal/constants/constants.go @@ -13,4 +13,10 @@ const ( ControlPlaneScalingGroupResourceName = "scalinggroup-controlplane" // WorkerScalingGroupResourceName resource name used for WorkerScaling. WorkerScalingGroupResourceName = "scalinggroup-worker" + // PlaceholderImageName name of the OS image used if upgrades are not yet supported. + PlaceholderImageName = "unsupportedCSP" + // PlaceholderControlPlaneScalingGroupName name of the control plane scaling group used if upgrades are not yet supported. + PlaceholderControlPlaneScalingGroupName = "control-planes-id" + // PlaceholderWorkerScalingGroupName name of the worker scaling group used if upgrades are not yet supported. + PlaceholderWorkerScalingGroupName = "workers-id" ) diff --git a/operators/constellation-node-operator/internal/deploy/BUILD.bazel b/operators/constellation-node-operator/internal/deploy/BUILD.bazel index e364535ac..417417d1e 100644 --- a/operators/constellation-node-operator/internal/deploy/BUILD.bazel +++ b/operators/constellation-node-operator/internal/deploy/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "@io_k8s_api//core/v1:core", "@io_k8s_apimachinery//pkg/api/errors", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_apimachinery//pkg/types", "@io_k8s_sigs_controller_runtime//pkg/client", "@io_k8s_sigs_controller_runtime//pkg/log", ], diff --git a/operators/constellation-node-operator/internal/deploy/deploy.go b/operators/constellation-node-operator/internal/deploy/deploy.go index 67a399f34..4e87878b7 100644 --- a/operators/constellation-node-operator/internal/deploy/deploy.go +++ b/operators/constellation-node-operator/internal/deploy/deploy.go @@ -19,6 +19,7 @@ import ( corev1 "k8s.io/api/core/v1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -26,6 +27,12 @@ import ( // InitialResources creates the initial resources for the node operator. func InitialResources(ctx context.Context, k8sClient client.Client, imageInfo imageInfoGetter, scalingGroupGetter scalingGroupGetter, uid string) error { logr := log.FromContext(ctx) + + if err := cleanupPlaceholders(ctx, k8sClient); err != nil { + return fmt.Errorf("cleaning up placeholder node version: %w", err) + } + logr.Info("cleaned up placeholders") + controlPlaneGroupIDs, workerGroupIDs, err := scalingGroupGetter.ListScalingGroups(ctx, uid) if err != nil { return fmt.Errorf("listing scaling groups: %w", err) @@ -138,6 +145,83 @@ func createNodeVersion(ctx context.Context, k8sClient client.Client, imageRefere return nil } +// cleanupPlaceholders deletes the existing resources from older operator versions if they are placeholders. +func cleanupPlaceholders(ctx context.Context, k8sClient client.Client) error { + if err := cleanupPlaceholderAutoscalingStrategy(ctx, k8sClient); err != nil { + return err + } + if err := cleanupPlaceholderScalingGroups(ctx, k8sClient); err != nil { + return err + } + return cleanupPlaceholderNodeVersion(ctx, k8sClient) +} + +func cleanupPlaceholderAutoscalingStrategy(ctx context.Context, k8sClient client.Client) error { + logr := log.FromContext(ctx) + autoscalingStrategy := &updatev1alpha1.AutoscalingStrategy{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: constants.AutoscalingStrategyResourceName}, autoscalingStrategy) + if k8sErrors.IsNotFound(err) { + logr.Info("no old autoscalingstrategy resource found - skipping cleanup", "name", constants.AutoscalingStrategyResourceName) + return nil + } else if err != nil { + logr.Info("cleaning up old autoscalingstrategy resource", "name", constants.AutoscalingStrategyResourceName, "error", err) + return err + } + if autoscalingStrategy.Spec.AutoscalerExtraArgs["cloud-provider"] != constants.PlaceholderImageName { + logr.Info("old autoscalingstrategy resource is not a placeholder - skipping cleanup", "name", constants.AutoscalingStrategyResourceName) + return nil + } + logr.Info("deleting old autoscalingstrategy resource", "name", constants.AutoscalingStrategyResourceName) + return k8sClient.Delete(ctx, autoscalingStrategy) +} + +// cleanupPlaceholderScalingGroups deletes the existing scalinggroup resource from older operator versions if they are placeholders. +func cleanupPlaceholderScalingGroups(ctx context.Context, k8sClient client.Client) error { + logr := log.FromContext(ctx) + names := []string{constants.PlaceholderControlPlaneScalingGroupName, constants.PlaceholderWorkerScalingGroupName} + for _, name := range names { + scalingGroup := &updatev1alpha1.ScalingGroup{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, scalingGroup) + if k8sErrors.IsNotFound(err) { + logr.Info("no old scalinggroup resource found - skipping cleanup", "name", name) + continue + } else if err != nil { + logr.Info("cleaning up old scalinggroup resource", "name", name, "error", err) + return err + } + if scalingGroup.Spec.AutoscalerGroupName != name || scalingGroup.Spec.GroupID != name { + logr.Info("real scalinggroup resource found - skipping cleanup", "name", name) + continue + } + logr.Info("cleaning up old scalinggroup resource") + if err := k8sClient.Delete(ctx, scalingGroup); err != nil { + logr.Info("cleaning up old scalinggroup resource", "name", name, "error", err) + return err + } + } + return nil +} + +// cleanupPlaceholder deletes the existing nodeversion resource from older operator versions if it was a placeholder. +func cleanupPlaceholderNodeVersion(ctx context.Context, k8sClient client.Client) error { + logr := log.FromContext(ctx) + nodeVersion := &updatev1alpha1.NodeVersion{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: mainconstants.NodeVersionResourceName}, nodeVersion) + if k8sErrors.IsNotFound(err) { + logr.Info("no old nodeversion resource found - skipping cleanup") + return nil + } else if err != nil { + logr.Info("cleaning up old nodeversion resource", "error", err) + return err + } + if nodeVersion.Spec.ImageReference != constants.PlaceholderImageName { + logr.Info("real nodeversion resource found - skipping cleanup") + return nil + } + logr.Info("cleaning up old nodeversion resource") + return k8sClient.Delete(ctx, nodeVersion) +} + // findLatestK8sComponentsConfigMap finds most recently created k8s-components configmap in the kube-system namespace. // It returns an error if there is no or multiple configmaps matching the prefix "k8s-components". func findLatestK8sComponentsConfigMap(ctx context.Context, k8sClient client.Client) (corev1.ConfigMap, error) {