operators: cleanup placeholder nodeversion (#1881)

* operators: cleanup placeholder nodeversion
* e2e: improve upgrade test portability

.github/workflows/e2e-upgrade.yml

@@ -184,6 +184,7 @@ jobs:
         env:
           KUBECONFIG: ${{ steps.e2e_test.outputs.kubeconfig }}
         run: |
+          kubectl logs -n kube-system -l "app.kubernetes.io/name=constellation-operator" --tail=-1 > node-operator.logs
           kubectl logs -n kube-system -l "app.kubernetes.io/name=node-maintenance-operator" --tail=-1 > node-maintenance-operator.logs
           kubectl get nodeversions.update.edgeless.systems constellation-version -o yaml > constellation-version.yaml
 
@@ -193,6 +194,7 @@ jobs:
         with:
           name: upgrade-logs
           path: |
+            node-operator.logs
             node-maintenance-operator.logs
             constellation-version.yaml
 

bazel/toolchains/go_module_deps.bzl

@@ -829,6 +829,14 @@ def go_dependencies():
         sum = "h1:XmPu4mXICgdGnC5dXGjUGbwUD/kUmS0l5Aop3LaevBM=",
         version = "v0.0.0-20230317132445-9c3c1fc0106e",
     )
+    go_repository(
+        name = "com_github_bazelbuild_rules_go",
+        build_file_generation = "on",
+        build_file_proto_mode = "disable_global",
+        importpath = "github.com/bazelbuild/rules_go",
+        sum = "h1:wkJLUDx59dntWMghuL8++GteoU1To6sRoKJXuyFtmf8=",
+        version = "v0.39.1",
+    )
 
     go_repository(
         name = "com_github_beeker1121_goque",

cli/internal/cmd/upgradeapply.go

@@ -45,7 +45,7 @@ func newUpgradeApplyCmd() *cobra.Command {
 	cmd.Flags().BoolP("yes", "y", false, "run upgrades without further confirmation\n"+
 		"WARNING: might delete your resources in case you are using cert-manager in your cluster. Please read the docs.\n"+
 		"WARNING: might unintentionally overwrite measurements in the running cluster.")
-	cmd.Flags().Duration("timeout", 3*time.Minute, "change helm upgrade timeout\n"+
+	cmd.Flags().Duration("timeout", 5*time.Minute, "change helm upgrade timeout\n"+
 		"Might be useful for slow connections or big clusters.")
 	if err := cmd.Flags().MarkHidden("timeout"); err != nil {
 		panic(err)

e2e/internal/upgrade/BUILD.bazel

@@ -29,11 +29,11 @@ go_test(
     # keep
     count = 1,
     data = [
-        "//cli:cli_enterprise_linux_amd64",
+        "//cli:cli_enterprise_host",
     ],
     embed = [":upgrade"],
     env = {
-        "PATH_CLI": "$(location //cli:cli_enterprise_linux_amd64)",
+        "PATH_CLI": "$(rlocationpath //cli:cli_enterprise_host)",
     },
     # keep
     gotags = ["e2e"],
@@ -48,6 +48,7 @@ go_test(
         "//internal/versions",
         "@com_github_spf13_afero//:afero",
         "@com_github_stretchr_testify//require",
+        "@io_bazel_rules_go//go/runfiles:go_default_library",
         "@io_k8s_api//core/v1:core",
         "@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
         "@io_k8s_client_go//kubernetes",

e2e/internal/upgrade/upgrade_test.go

@@ -17,11 +17,13 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/bazelbuild/rules_go/go/runfiles"
 	"github.com/edgelesssys/constellation/v2/e2e/internal/kubectl"
 	attestationconfigfetcher "github.com/edgelesssys/constellation/v2/internal/api/attestationconfig/fetcher"
 	"github.com/edgelesssys/constellation/v2/internal/config"
@@ -68,6 +70,7 @@ func TestUpgrade(t *testing.T) {
 
 	require.NotEqual(*targetImage, "", "--target-image needs to be specified")
 
+	log.Println("Waiting for nodes and pods to be ready.")
 	testNodesEventuallyAvailable(t, k, *wantControl, *wantWorker)
 	testPodsEventuallyReady(t, k, "kube-system")
 
@@ -75,6 +78,7 @@ func TestUpgrade(t *testing.T) {
 	require.NoError(err)
 
 	// Migrate config if necessary.
+	log.Println("Migrating config if needed.")
 	cmd := exec.CommandContext(context.Background(), cli, "config", "migrate", "--config", constants.ConfigFilename, "--force", "--debug")
 	stdout, stderr, err := runCommandWithSeparateOutputs(cmd)
 	require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr))
@@ -138,16 +142,35 @@ func workingDir(workspace string) (string, error) {
 }
 
 // getCLIPath returns the path to the CLI.
-func getCLIPath(cliPath string) (string, error) {
+func getCLIPath(cliPathFlag string) (string, error) {
 	pathCLI := os.Getenv("PATH_CLI")
+	var relCLIPath string
 	switch {
 	case pathCLI != "":
-		return pathCLI, nil
+		relCLIPath = pathCLI
-	case cliPath != "":
+	case cliPathFlag != "":
-		return cliPath, nil
+		relCLIPath = cliPathFlag
 	default:
 		return "", errors.New("neither 'PATH_CLI' nor 'cli' flag set")
 	}
+
+	// try to find the CLI in the working directory
+	// (e.g. when running via `go test` or when specifying a path manually)
+	workdir, err := os.Getwd()
+	if err != nil {
+		return "", fmt.Errorf("getting working directory: %w", err)
+	}
+
+	absCLIPath := relCLIPath
+	if !filepath.IsAbs(relCLIPath) {
+		absCLIPath = filepath.Join(workdir, relCLIPath)
+	}
+	if _, err := os.Stat(absCLIPath); err == nil {
+		return absCLIPath, nil
+	}
+
+	// fall back to runfiles (e.g. when running via bazel)
+	return runfiles.Rlocation(pathCLI)
 }
 
 // testPodsEventuallyReady checks that:

go.mod

@@ -177,6 +177,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.18.11 // indirect
+	github.com/bazelbuild/rules_go v0.39.1
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect

go.sum

@@ -295,6 +295,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.18.11/go.mod h1:BgQOMsg8av8jset59jel
 github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
+github.com/bazelbuild/rules_go v0.39.1 h1:wkJLUDx59dntWMghuL8++GteoU1To6sRoKJXuyFtmf8=
+github.com/bazelbuild/rules_go v0.39.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=

operators/constellation-node-operator/internal/cloud/fake/client/BUILD.bazel

@@ -5,5 +5,8 @@ go_library(
     srcs = ["client.go"],
     importpath = "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/fake/client",
     visibility = ["//operators/constellation-node-operator:__subpackages__"],
-    deps = ["//operators/constellation-node-operator/api/v1alpha1"],
+    deps = [
+        "//operators/constellation-node-operator/api/v1alpha1",
+        "//operators/constellation-node-operator/internal/constants",
+    ],
 )

operators/constellation-node-operator/internal/cloud/fake/client/client.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 
 	updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
+	"github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/constants"
 )
 
 const (
@@ -53,7 +54,7 @@ func (c *Client) SetScalingGroupImage(_ context.Context, _, _ string) error {
 
 // GetScalingGroupImage retrieves the image currently used by a scaling group.
 func (c *Client) GetScalingGroupImage(_ context.Context, _ string) (string, error) {
-	return "unsupportedCSP", nil
+	return constants.PlaceholderImageName, nil
 }
 
 // GetScalingGroupName retrieves the name of a scaling group.
@@ -87,5 +88,5 @@ func (c *Client) ListScalingGroups(_ context.Context, _ string) (controlPlaneGro
 
 // AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler.
 func (c *Client) AutoscalingCloudProvider() string {
-	return "unsupportedCSP"
+	return constants.PlaceholderImageName
 }

operators/constellation-node-operator/internal/constants/constants.go

@@ -13,4 +13,10 @@ const (
 	ControlPlaneScalingGroupResourceName = "scalinggroup-controlplane"
 	// WorkerScalingGroupResourceName resource name used for WorkerScaling.
 	WorkerScalingGroupResourceName = "scalinggroup-worker"
+	// PlaceholderImageName name of the OS image used if upgrades are not yet supported.
+	PlaceholderImageName = "unsupportedCSP"
+	// PlaceholderControlPlaneScalingGroupName name of the control plane scaling group used if upgrades are not yet supported.
+	PlaceholderControlPlaneScalingGroupName = "control-planes-id"
+	// PlaceholderWorkerScalingGroupName name of the worker scaling group used if upgrades are not yet supported.
+	PlaceholderWorkerScalingGroupName = "workers-id"
 )

operators/constellation-node-operator/internal/deploy/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "@io_k8s_api//core/v1:core",
         "@io_k8s_apimachinery//pkg/api/errors",
         "@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
+        "@io_k8s_apimachinery//pkg/types",
         "@io_k8s_sigs_controller_runtime//pkg/client",
         "@io_k8s_sigs_controller_runtime//pkg/log",
     ],

operators/constellation-node-operator/internal/deploy/deploy.go

@@ -19,6 +19,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
@@ -26,6 +27,12 @@ import (
 // InitialResources creates the initial resources for the node operator.
 func InitialResources(ctx context.Context, k8sClient client.Client, imageInfo imageInfoGetter, scalingGroupGetter scalingGroupGetter, uid string) error {
 	logr := log.FromContext(ctx)
+
+	if err := cleanupPlaceholders(ctx, k8sClient); err != nil {
+		return fmt.Errorf("cleaning up placeholder node version: %w", err)
+	}
+	logr.Info("cleaned up placeholders")
+
 	controlPlaneGroupIDs, workerGroupIDs, err := scalingGroupGetter.ListScalingGroups(ctx, uid)
 	if err != nil {
 		return fmt.Errorf("listing scaling groups: %w", err)
@@ -138,6 +145,83 @@ func createNodeVersion(ctx context.Context, k8sClient client.Client, imageRefere
 	return nil
 }
 
+// cleanupPlaceholders deletes the existing resources from older operator versions if they are placeholders.
+func cleanupPlaceholders(ctx context.Context, k8sClient client.Client) error {
+	if err := cleanupPlaceholderAutoscalingStrategy(ctx, k8sClient); err != nil {
+		return err
+	}
+	if err := cleanupPlaceholderScalingGroups(ctx, k8sClient); err != nil {
+		return err
+	}
+	return cleanupPlaceholderNodeVersion(ctx, k8sClient)
+}
+
+func cleanupPlaceholderAutoscalingStrategy(ctx context.Context, k8sClient client.Client) error {
+	logr := log.FromContext(ctx)
+	autoscalingStrategy := &updatev1alpha1.AutoscalingStrategy{}
+	err := k8sClient.Get(ctx, types.NamespacedName{Name: constants.AutoscalingStrategyResourceName}, autoscalingStrategy)
+	if k8sErrors.IsNotFound(err) {
+		logr.Info("no old autoscalingstrategy resource found - skipping cleanup", "name", constants.AutoscalingStrategyResourceName)
+		return nil
+	} else if err != nil {
+		logr.Info("cleaning up old autoscalingstrategy resource", "name", constants.AutoscalingStrategyResourceName, "error", err)
+		return err
+	}
+	if autoscalingStrategy.Spec.AutoscalerExtraArgs["cloud-provider"] != constants.PlaceholderImageName {
+		logr.Info("old autoscalingstrategy resource is not a placeholder - skipping cleanup", "name", constants.AutoscalingStrategyResourceName)
+		return nil
+	}
+	logr.Info("deleting old autoscalingstrategy resource", "name", constants.AutoscalingStrategyResourceName)
+	return k8sClient.Delete(ctx, autoscalingStrategy)
+}
+
+// cleanupPlaceholderScalingGroups deletes the existing scalinggroup resource from older operator versions if they are placeholders.
+func cleanupPlaceholderScalingGroups(ctx context.Context, k8sClient client.Client) error {
+	logr := log.FromContext(ctx)
+	names := []string{constants.PlaceholderControlPlaneScalingGroupName, constants.PlaceholderWorkerScalingGroupName}
+	for _, name := range names {
+		scalingGroup := &updatev1alpha1.ScalingGroup{}
+		err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, scalingGroup)
+		if k8sErrors.IsNotFound(err) {
+			logr.Info("no old scalinggroup resource found - skipping cleanup", "name", name)
+			continue
+		} else if err != nil {
+			logr.Info("cleaning up old scalinggroup resource", "name", name, "error", err)
+			return err
+		}
+		if scalingGroup.Spec.AutoscalerGroupName != name || scalingGroup.Spec.GroupID != name {
+			logr.Info("real scalinggroup resource found - skipping cleanup", "name", name)
+			continue
+		}
+		logr.Info("cleaning up old scalinggroup resource")
+		if err := k8sClient.Delete(ctx, scalingGroup); err != nil {
+			logr.Info("cleaning up old scalinggroup resource", "name", name, "error", err)
+			return err
+		}
+	}
+	return nil
+}
+
+// cleanupPlaceholder deletes the existing nodeversion resource from older operator versions if it was a placeholder.
+func cleanupPlaceholderNodeVersion(ctx context.Context, k8sClient client.Client) error {
+	logr := log.FromContext(ctx)
+	nodeVersion := &updatev1alpha1.NodeVersion{}
+	err := k8sClient.Get(ctx, types.NamespacedName{Name: mainconstants.NodeVersionResourceName}, nodeVersion)
+	if k8sErrors.IsNotFound(err) {
+		logr.Info("no old nodeversion resource found - skipping cleanup")
+		return nil
+	} else if err != nil {
+		logr.Info("cleaning up old nodeversion resource", "error", err)
+		return err
+	}
+	if nodeVersion.Spec.ImageReference != constants.PlaceholderImageName {
+		logr.Info("real nodeversion resource found - skipping cleanup")
+		return nil
+	}
+	logr.Info("cleaning up old nodeversion resource")
+	return k8sClient.Delete(ctx, nodeVersion)
+}
+
 // findLatestK8sComponentsConfigMap finds most recently created k8s-components configmap in the kube-system namespace.
 // It returns an error if there is no or multiple configmaps matching the prefix "k8s-components".
 func findLatestK8sComponentsConfigMap(ctx context.Context, k8sClient client.Client) (corev1.ConfigMap, error) {