operators: infrastructure autodiscovery (#1958)

* helm: configure GCP cloud controller manager to search in all zones of a region

See also: d716fdd452/providers/gce/gce.go (L376-L380)

* operators: add nodeGroupName to ScalingGroup CRD

NodeGroupName is the human friendly name of the node group that will be exposed to customers via the Constellation config in the future.

* operators: support simple executor / scheduler to reconcile on non-k8s resources

* operators: add new return type for ListScalingGroups to support arbitrary node groups

* operators: ListScalingGroups should return additionally created node groups on AWS

* operators: ListScalingGroups should return additionally created node groups on Azure

* operators: ListScalingGroups should return additionally created node groups on GCP

* operators: ListScalingGroups should return additionally created node groups on unsupported CSPs

* operators: implement external scaling group reconciler

This controller scans the cloud provider infrastructure and changes k8s resources accordingly.
It creates ScaleSet resources when new node groups are created and deletes them if the node groups are removed.

* operators: no longer create scale sets when the operator starts

In the future, scale sets are created dynamically.

* operators: watch for node join/leave events using a controller

* operators: deploy new controllers

* docs: update auto scaling documentation with support for node groups

operators/constellation-node-operator/internal/cloud/aws/client/BUILD.bazel

@@ -15,6 +15,7 @@ go_library(
     visibility = ["//operators/constellation-node-operator:__subpackages__"],
     deps = [
         "//operators/constellation-node-operator/api/v1alpha1",
+        "//operators/constellation-node-operator/internal/cloud/api",
         "@com_github_aws_aws_sdk_go_v2_config//:config",
         "@com_github_aws_aws_sdk_go_v2_feature_ec2_imds//:imds",
         "@com_github_aws_aws_sdk_go_v2_service_autoscaling//:autoscaling",
@@ -36,6 +37,7 @@ go_test(
     embed = [":client"],
     deps = [
         "//operators/constellation-node-operator/api/v1alpha1",
+        "//operators/constellation-node-operator/internal/cloud/api",
         "@com_github_aws_aws_sdk_go_v2_service_autoscaling//:autoscaling",
         "@com_github_aws_aws_sdk_go_v2_service_autoscaling//types",
         "@com_github_aws_aws_sdk_go_v2_service_ec2//:ec2",

operators/constellation-node-operator/internal/cloud/aws/client/scalinggroup.go

@@ -15,6 +15,8 @@ import (
 	scalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
+	cspapi "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api"
 )
 
 // GetScalingGroupImage returns the image URI of the scaling group.
@@ -140,7 +142,8 @@ func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error)
 }
 
 // ListScalingGroups retrieves a list of scaling groups for the cluster.
-func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) {
+func (c *Client) ListScalingGroups(ctx context.Context, uid string) ([]cspapi.ScalingGroup, error) {
+	results := []cspapi.ScalingGroup{}
 	output, err := c.scalingClient.DescribeAutoScalingGroups(
 		ctx,
 		&autoscaling.DescribeAutoScalingGroupsInput{
@@ -153,22 +156,62 @@ func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlan
 		},
 	)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to describe scaling groups: %w", err)
+		return nil, fmt.Errorf("failed to describe scaling groups: %w", err)
 	}
 
 	for _, group := range output.AutoScalingGroups {
 		if group.Tags == nil {
 			continue
 		}
+
+		var role updatev1alpha1.NodeRole
+		var nodeGroupName string
 		for _, tag := range group.Tags {
-			if *tag.Key == "constellation-role" {
-				if *tag.Value == "control-plane" {
-					controlPlaneGroupIDs = append(controlPlaneGroupIDs, *group.AutoScalingGroupName)
-				} else if *tag.Value == "worker" {
-					workerGroupIDs = append(workerGroupIDs, *group.AutoScalingGroupName)
-				}
+			if tag.Key == nil || tag.Value == nil {
+				continue
+			}
+			key := *tag.Key
+			switch key {
+			case "constellation-role":
+				role = updatev1alpha1.NodeRoleFromString(*tag.Value)
+			case "constellation-node-group":
+				nodeGroupName = *tag.Value
 			}
 		}
+
+		// fallback for legacy clusters
+		// TODO(malt3): remove this fallback once we can assume all clusters have the correct labels
+		if nodeGroupName == "" {
+			switch role {
+			case updatev1alpha1.ControlPlaneRole:
+				nodeGroupName = "control_plane_default"
+			case updatev1alpha1.WorkerRole:
+				nodeGroupName = "worker_default"
+			}
+		}
+
+		name, err := c.GetScalingGroupName(*group.AutoScalingGroupName)
+		if err != nil {
+			return nil, fmt.Errorf("getting scaling group name: %w", err)
+		}
+
+		nodeGroupName, err = c.GetScalingGroupName(nodeGroupName)
+		if err != nil {
+			return nil, fmt.Errorf("getting node group name: %w", err)
+		}
+
+		autoscalerGroupName, err := c.GetAutoscalingGroupName(*group.AutoScalingGroupName)
+		if err != nil {
+			return nil, fmt.Errorf("getting autoscaler group name: %w", err)
+		}
+
+		results = append(results, cspapi.ScalingGroup{
+			Name:                 name,
+			NodeGroupName:        nodeGroupName,
+			GroupID:              *group.AutoScalingGroupName,
+			AutoscalingGroupName: autoscalerGroupName,
+			Role:                 role,
+		})
 	}
-	return controlPlaneGroupIDs, workerGroupIDs, nil
+	return results, nil
 }

operators/constellation-node-operator/internal/cloud/aws/client/scalinggroup_test.go

@@ -14,6 +14,7 @@ import (
 	scalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	cspapi "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -229,8 +230,7 @@ func TestListScalingGroups(t *testing.T) {
 		providerID                   string
 		describeAutoScalingGroupsOut []*autoscaling.DescribeAutoScalingGroupsOutput
 		describeAutoScalingGroupsErr []error
-		wantControlPlaneGroupIDs     []string
-		wantWorkerGroupIDs           []string
+		wantGroups                   []cspapi.ScalingGroup
 		wantErr                      bool
 	}{
 		"listing scaling groups work": {
@@ -263,6 +263,10 @@ func TestListScalingGroups(t *testing.T) {
 									Key:   toPtr("constellation-role"),
 									Value: toPtr("worker"),
 								},
+								{
+									Key:   toPtr("constellation-node-group"),
+									Value: toPtr("foo-group"),
+								},
 							},
 						},
 						{
@@ -272,8 +276,29 @@ func TestListScalingGroups(t *testing.T) {
 				},
 			},
 			describeAutoScalingGroupsErr: []error{nil},
-			wantControlPlaneGroupIDs:     []string{"control-plane-asg"},
-			wantWorkerGroupIDs:           []string{"worker-asg", "worker-asg-2"},
+			wantGroups: []cspapi.ScalingGroup{
+				{
+					Name:                 "control-plane-asg",
+					NodeGroupName:        "control_plane_default",
+					GroupID:              "control-plane-asg",
+					AutoscalingGroupName: "control-plane-asg",
+					Role:                 "ControlPlane",
+				},
+				{
+					Name:                 "worker-asg",
+					NodeGroupName:        "worker_default",
+					GroupID:              "worker-asg",
+					AutoscalingGroupName: "worker-asg",
+					Role:                 "Worker",
+				},
+				{
+					Name:                 "worker-asg-2",
+					NodeGroupName:        "foo-group",
+					GroupID:              "worker-asg-2",
+					AutoscalingGroupName: "worker-asg-2",
+					Role:                 "Worker",
+				},
+			},
 		},
 		"fails when describing scaling groups fails": {
 			providerID:                   "aws:///us-east-2a/i-00000000000000000",
@@ -293,14 +318,13 @@ func TestListScalingGroups(t *testing.T) {
 					describeAutoScalingGroupsErr: tc.describeAutoScalingGroupsErr,
 				},
 			}
-			controlPlaneGroupIDs, workerGroupIDs, err := client.ListScalingGroups(context.Background(), tc.providerID)
+			gotGroups, err := client.ListScalingGroups(context.Background(), tc.providerID)
 			if tc.wantErr {
 				assert.Error(err)
 				return
 			}
 			require.NoError(err)
-			assert.Equal(tc.wantControlPlaneGroupIDs, controlPlaneGroupIDs)
-			assert.Equal(tc.wantWorkerGroupIDs, workerGroupIDs)
+			assert.Equal(tc.wantGroups, gotGroups)
 		})
 	}
 }