operators: cleanup placeholder nodeversion (#1881)

* operators: cleanup placeholder nodeversion
* e2e: improve upgrade test portability

operators/constellation-node-operator/internal/cloud/fake/client/BUILD.bazel

@@ -5,5 +5,8 @@ go_library(
     srcs = ["client.go"],
     importpath = "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/fake/client",
     visibility = ["//operators/constellation-node-operator:__subpackages__"],
-    deps = ["//operators/constellation-node-operator/api/v1alpha1"],
+    deps = [
+        "//operators/constellation-node-operator/api/v1alpha1",
+        "//operators/constellation-node-operator/internal/constants",
+    ],
 )

operators/constellation-node-operator/internal/cloud/fake/client/client.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 
 	updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
+	"github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/constants"
 )
 
 const (
@@ -53,7 +54,7 @@ func (c *Client) SetScalingGroupImage(_ context.Context, _, _ string) error {
 
 // GetScalingGroupImage retrieves the image currently used by a scaling group.
 func (c *Client) GetScalingGroupImage(_ context.Context, _ string) (string, error) {
-	return "unsupportedCSP", nil
+	return constants.PlaceholderImageName, nil
 }
 
 // GetScalingGroupName retrieves the name of a scaling group.
@@ -87,5 +88,5 @@ func (c *Client) ListScalingGroups(_ context.Context, _ string) (controlPlaneGro
 
 // AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler.
 func (c *Client) AutoscalingCloudProvider() string {
-	return "unsupportedCSP"
+	return constants.PlaceholderImageName
 }

operators/constellation-node-operator/internal/constants/constants.go

@@ -13,4 +13,10 @@ const (
 	ControlPlaneScalingGroupResourceName = "scalinggroup-controlplane"
 	// WorkerScalingGroupResourceName resource name used for WorkerScaling.
 	WorkerScalingGroupResourceName = "scalinggroup-worker"
+	// PlaceholderImageName name of the OS image used if upgrades are not yet supported.
+	PlaceholderImageName = "unsupportedCSP"
+	// PlaceholderControlPlaneScalingGroupName name of the control plane scaling group used if upgrades are not yet supported.
+	PlaceholderControlPlaneScalingGroupName = "control-planes-id"
+	// PlaceholderWorkerScalingGroupName name of the worker scaling group used if upgrades are not yet supported.
+	PlaceholderWorkerScalingGroupName = "workers-id"
 )

operators/constellation-node-operator/internal/deploy/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "@io_k8s_api//core/v1:core",
         "@io_k8s_apimachinery//pkg/api/errors",
         "@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
+        "@io_k8s_apimachinery//pkg/types",
         "@io_k8s_sigs_controller_runtime//pkg/client",
         "@io_k8s_sigs_controller_runtime//pkg/log",
     ],

operators/constellation-node-operator/internal/deploy/deploy.go

@@ -19,6 +19,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
@@ -26,6 +27,12 @@ import (
 // InitialResources creates the initial resources for the node operator.
 func InitialResources(ctx context.Context, k8sClient client.Client, imageInfo imageInfoGetter, scalingGroupGetter scalingGroupGetter, uid string) error {
 	logr := log.FromContext(ctx)
+
+	if err := cleanupPlaceholders(ctx, k8sClient); err != nil {
+		return fmt.Errorf("cleaning up placeholder node version: %w", err)
+	}
+	logr.Info("cleaned up placeholders")
+
 	controlPlaneGroupIDs, workerGroupIDs, err := scalingGroupGetter.ListScalingGroups(ctx, uid)
 	if err != nil {
 		return fmt.Errorf("listing scaling groups: %w", err)
@@ -138,6 +145,83 @@ func createNodeVersion(ctx context.Context, k8sClient client.Client, imageRefere
 	return nil
 }
 
+// cleanupPlaceholders deletes the existing resources from older operator versions if they are placeholders.
+func cleanupPlaceholders(ctx context.Context, k8sClient client.Client) error {
+	if err := cleanupPlaceholderAutoscalingStrategy(ctx, k8sClient); err != nil {
+		return err
+	}
+	if err := cleanupPlaceholderScalingGroups(ctx, k8sClient); err != nil {
+		return err
+	}
+	return cleanupPlaceholderNodeVersion(ctx, k8sClient)
+}
+
+func cleanupPlaceholderAutoscalingStrategy(ctx context.Context, k8sClient client.Client) error {
+	logr := log.FromContext(ctx)
+	autoscalingStrategy := &updatev1alpha1.AutoscalingStrategy{}
+	err := k8sClient.Get(ctx, types.NamespacedName{Name: constants.AutoscalingStrategyResourceName}, autoscalingStrategy)
+	if k8sErrors.IsNotFound(err) {
+		logr.Info("no old autoscalingstrategy resource found - skipping cleanup", "name", constants.AutoscalingStrategyResourceName)
+		return nil
+	} else if err != nil {
+		logr.Info("cleaning up old autoscalingstrategy resource", "name", constants.AutoscalingStrategyResourceName, "error", err)
+		return err
+	}
+	if autoscalingStrategy.Spec.AutoscalerExtraArgs["cloud-provider"] != constants.PlaceholderImageName {
+		logr.Info("old autoscalingstrategy resource is not a placeholder - skipping cleanup", "name", constants.AutoscalingStrategyResourceName)
+		return nil
+	}
+	logr.Info("deleting old autoscalingstrategy resource", "name", constants.AutoscalingStrategyResourceName)
+	return k8sClient.Delete(ctx, autoscalingStrategy)
+}
+
+// cleanupPlaceholderScalingGroups deletes the existing scalinggroup resource from older operator versions if they are placeholders.
+func cleanupPlaceholderScalingGroups(ctx context.Context, k8sClient client.Client) error {
+	logr := log.FromContext(ctx)
+	names := []string{constants.PlaceholderControlPlaneScalingGroupName, constants.PlaceholderWorkerScalingGroupName}
+	for _, name := range names {
+		scalingGroup := &updatev1alpha1.ScalingGroup{}
+		err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, scalingGroup)
+		if k8sErrors.IsNotFound(err) {
+			logr.Info("no old scalinggroup resource found - skipping cleanup", "name", name)
+			continue
+		} else if err != nil {
+			logr.Info("cleaning up old scalinggroup resource", "name", name, "error", err)
+			return err
+		}
+		if scalingGroup.Spec.AutoscalerGroupName != name || scalingGroup.Spec.GroupID != name {
+			logr.Info("real scalinggroup resource found - skipping cleanup", "name", name)
+			continue
+		}
+		logr.Info("cleaning up old scalinggroup resource")
+		if err := k8sClient.Delete(ctx, scalingGroup); err != nil {
+			logr.Info("cleaning up old scalinggroup resource", "name", name, "error", err)
+			return err
+		}
+	}
+	return nil
+}
+
+// cleanupPlaceholder deletes the existing nodeversion resource from older operator versions if it was a placeholder.
+func cleanupPlaceholderNodeVersion(ctx context.Context, k8sClient client.Client) error {
+	logr := log.FromContext(ctx)
+	nodeVersion := &updatev1alpha1.NodeVersion{}
+	err := k8sClient.Get(ctx, types.NamespacedName{Name: mainconstants.NodeVersionResourceName}, nodeVersion)
+	if k8sErrors.IsNotFound(err) {
+		logr.Info("no old nodeversion resource found - skipping cleanup")
+		return nil
+	} else if err != nil {
+		logr.Info("cleaning up old nodeversion resource", "error", err)
+		return err
+	}
+	if nodeVersion.Spec.ImageReference != constants.PlaceholderImageName {
+		logr.Info("real nodeversion resource found - skipping cleanup")
+		return nil
+	}
+	logr.Info("cleaning up old nodeversion resource")
+	return k8sClient.Delete(ctx, nodeVersion)
+}
+
 // findLatestK8sComponentsConfigMap finds most recently created k8s-components configmap in the kube-system namespace.
 // It returns an error if there is no or multiple configmaps matching the prefix "k8s-components".
 func findLatestK8sComponentsConfigMap(ctx context.Context, k8sClient client.Client) (corev1.ConfigMap, error) {