mirror of
https://github.com/edgelesssys/constellation.git
synced 2025-06-28 07:57:23 -04:00
operators: infrastructure autodiscovery (#1958)
* helm: configure GCP cloud controller manager to search in all zones of a region
See also: d716fdd452/providers/gce/gce.go (L376-L380)
* operators: add nodeGroupName to ScalingGroup CRD
NodeGroupName is the human friendly name of the node group that will be exposed to customers via the Constellation config in the future.
* operators: support simple executor / scheduler to reconcile on non-k8s resources
* operators: add new return type for ListScalingGroups to support arbitrary node groups
* operators: ListScalingGroups should return additionally created node groups on AWS
* operators: ListScalingGroups should return additionally created node groups on Azure
* operators: ListScalingGroups should return additionally created node groups on GCP
* operators: ListScalingGroups should return additionally created node groups on unsupported CSPs
* operators: implement external scaling group reconciler
This controller scans the cloud provider infrastructure and changes k8s resources accordingly.
It creates ScaleSet resources when new node groups are created and deletes them if the node groups are removed.
* operators: no longer create scale sets when the operator starts
In the future, scale sets are created dynamically.
* operators: watch for node join/leave events using a controller
* operators: deploy new controllers
* docs: update auto scaling documentation with support for node groups
This commit is contained in:
Malte Poll
Adrian Stobbe
parent
10a540c290
commit
388ff011a3
36 changed files with 1836 additions and 232 deletions
|
|
@ -0,0 +1,9 @@
|
|||
load("@io_bazel_rules_go//go:def.bzl", "go_library")
|
||||
|
||||
go_library(
|
||||
name = "api",
|
||||
srcs = ["scalinggroup.go"],
|
||||
importpath = "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api",
|
||||
visibility = ["//operators/constellation-node-operator:__subpackages__"],
|
||||
deps = ["//operators/constellation-node-operator/api/v1alpha1"],
|
||||
)
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package api
|
||||
|
||||
import updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
|
||||
|
||||
// ScalingGroup is a cloud provider scaling group.
|
||||
type ScalingGroup struct {
|
||||
// Name is the csp specific name of the scaling group.
|
||||
Name string
|
||||
// NodeGroupName is the human friendly name of the node group
|
||||
// as defined in the Constellation configuration.
|
||||
NodeGroupName string
|
||||
// GroupID is the CSP specific, canonical identifier of a scaling group.
|
||||
GroupID string
|
||||
// AutoscalingGroupName is name that is expected by the autoscaler.
|
||||
AutoscalingGroupName string
|
||||
// Role is the role of the nodes in the scaling group.
|
||||
Role updatev1alpha1.NodeRole
|
||||
}
|
||||
|
|
@ -15,6 +15,7 @@ go_library(
|
|||
visibility = ["//operators/constellation-node-operator:__subpackages__"],
|
||||
deps = [
|
||||
"//operators/constellation-node-operator/api/v1alpha1",
|
||||
"//operators/constellation-node-operator/internal/cloud/api",
|
||||
"@com_github_aws_aws_sdk_go_v2_config//:config",
|
||||
"@com_github_aws_aws_sdk_go_v2_feature_ec2_imds//:imds",
|
||||
"@com_github_aws_aws_sdk_go_v2_service_autoscaling//:autoscaling",
|
||||
|
|
@ -36,6 +37,7 @@ go_test(
|
|||
embed = [":client"],
|
||||
deps = [
|
||||
"//operators/constellation-node-operator/api/v1alpha1",
|
||||
"//operators/constellation-node-operator/internal/cloud/api",
|
||||
"@com_github_aws_aws_sdk_go_v2_service_autoscaling//:autoscaling",
|
||||
"@com_github_aws_aws_sdk_go_v2_service_autoscaling//types",
|
||||
"@com_github_aws_aws_sdk_go_v2_service_ec2//:ec2",
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ import (
|
|||
scalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
|
||||
"github.com/aws/aws-sdk-go-v2/service/ec2"
|
||||
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
|
||||
updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
|
||||
cspapi "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api"
|
||||
)
|
||||
|
||||
// GetScalingGroupImage returns the image URI of the scaling group.
|
||||
|
|
@ -140,7 +142,8 @@ func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error)
|
|||
}
|
||||
|
||||
// ListScalingGroups retrieves a list of scaling groups for the cluster.
|
||||
func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) {
|
||||
func (c *Client) ListScalingGroups(ctx context.Context, uid string) ([]cspapi.ScalingGroup, error) {
|
||||
results := []cspapi.ScalingGroup{}
|
||||
output, err := c.scalingClient.DescribeAutoScalingGroups(
|
||||
ctx,
|
||||
&autoscaling.DescribeAutoScalingGroupsInput{
|
||||
|
|
@ -153,22 +156,62 @@ func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlan
|
|||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to describe scaling groups: %w", err)
|
||||
return nil, fmt.Errorf("failed to describe scaling groups: %w", err)
|
||||
}
|
||||
|
||||
for _, group := range output.AutoScalingGroups {
|
||||
if group.Tags == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var role updatev1alpha1.NodeRole
|
||||
var nodeGroupName string
|
||||
for _, tag := range group.Tags {
|
||||
if *tag.Key == "constellation-role" {
|
||||
if *tag.Value == "control-plane" {
|
||||
controlPlaneGroupIDs = append(controlPlaneGroupIDs, *group.AutoScalingGroupName)
|
||||
} else if *tag.Value == "worker" {
|
||||
workerGroupIDs = append(workerGroupIDs, *group.AutoScalingGroupName)
|
||||
}
|
||||
if tag.Key == nil || tag.Value == nil {
|
||||
continue
|
||||
}
|
||||
key := *tag.Key
|
||||
switch key {
|
||||
case "constellation-role":
|
||||
role = updatev1alpha1.NodeRoleFromString(*tag.Value)
|
||||
case "constellation-node-group":
|
||||
nodeGroupName = *tag.Value
|
||||
}
|
||||
}
|
||||
|
||||
// fallback for legacy clusters
|
||||
// TODO(malt3): remove this fallback once we can assume all clusters have the correct labels
|
||||
if nodeGroupName == "" {
|
||||
switch role {
|
||||
case updatev1alpha1.ControlPlaneRole:
|
||||
nodeGroupName = "control_plane_default"
|
||||
case updatev1alpha1.WorkerRole:
|
||||
nodeGroupName = "worker_default"
|
||||
}
|
||||
}
|
||||
|
||||
name, err := c.GetScalingGroupName(*group.AutoScalingGroupName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting scaling group name: %w", err)
|
||||
}
|
||||
|
||||
nodeGroupName, err = c.GetScalingGroupName(nodeGroupName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting node group name: %w", err)
|
||||
}
|
||||
|
||||
autoscalerGroupName, err := c.GetAutoscalingGroupName(*group.AutoScalingGroupName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting autoscaler group name: %w", err)
|
||||
}
|
||||
|
||||
results = append(results, cspapi.ScalingGroup{
|
||||
Name: name,
|
||||
NodeGroupName: nodeGroupName,
|
||||
GroupID: *group.AutoScalingGroupName,
|
||||
AutoscalingGroupName: autoscalerGroupName,
|
||||
Role: role,
|
||||
})
|
||||
}
|
||||
return controlPlaneGroupIDs, workerGroupIDs, nil
|
||||
return results, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import (
|
|||
scalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
|
||||
"github.com/aws/aws-sdk-go-v2/service/ec2"
|
||||
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
|
||||
cspapi "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -229,8 +230,7 @@ func TestListScalingGroups(t *testing.T) {
|
|||
providerID string
|
||||
describeAutoScalingGroupsOut []*autoscaling.DescribeAutoScalingGroupsOutput
|
||||
describeAutoScalingGroupsErr []error
|
||||
wantControlPlaneGroupIDs []string
|
||||
wantWorkerGroupIDs []string
|
||||
wantGroups []cspapi.ScalingGroup
|
||||
wantErr bool
|
||||
}{
|
||||
"listing scaling groups work": {
|
||||
|
|
@ -263,6 +263,10 @@ func TestListScalingGroups(t *testing.T) {
|
|||
Key: toPtr("constellation-role"),
|
||||
Value: toPtr("worker"),
|
||||
},
|
||||
{
|
||||
Key: toPtr("constellation-node-group"),
|
||||
Value: toPtr("foo-group"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -272,8 +276,29 @@ func TestListScalingGroups(t *testing.T) {
|
|||
},
|
||||
},
|
||||
describeAutoScalingGroupsErr: []error{nil},
|
||||
wantControlPlaneGroupIDs: []string{"control-plane-asg"},
|
||||
wantWorkerGroupIDs: []string{"worker-asg", "worker-asg-2"},
|
||||
wantGroups: []cspapi.ScalingGroup{
|
||||
{
|
||||
Name: "control-plane-asg",
|
||||
NodeGroupName: "control_plane_default",
|
||||
GroupID: "control-plane-asg",
|
||||
AutoscalingGroupName: "control-plane-asg",
|
||||
Role: "ControlPlane",
|
||||
},
|
||||
{
|
||||
Name: "worker-asg",
|
||||
NodeGroupName: "worker_default",
|
||||
GroupID: "worker-asg",
|
||||
AutoscalingGroupName: "worker-asg",
|
||||
Role: "Worker",
|
||||
},
|
||||
{
|
||||
Name: "worker-asg-2",
|
||||
NodeGroupName: "foo-group",
|
||||
GroupID: "worker-asg-2",
|
||||
AutoscalingGroupName: "worker-asg-2",
|
||||
Role: "Worker",
|
||||
},
|
||||
},
|
||||
},
|
||||
"fails when describing scaling groups fails": {
|
||||
providerID: "aws:///us-east-2a/i-00000000000000000",
|
||||
|
|
@ -293,14 +318,13 @@ func TestListScalingGroups(t *testing.T) {
|
|||
describeAutoScalingGroupsErr: tc.describeAutoScalingGroupsErr,
|
||||
},
|
||||
}
|
||||
controlPlaneGroupIDs, workerGroupIDs, err := client.ListScalingGroups(context.Background(), tc.providerID)
|
||||
gotGroups, err := client.ListScalingGroups(context.Background(), tc.providerID)
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
return
|
||||
}
|
||||
require.NoError(err)
|
||||
assert.Equal(tc.wantControlPlaneGroupIDs, controlPlaneGroupIDs)
|
||||
assert.Equal(tc.wantWorkerGroupIDs, workerGroupIDs)
|
||||
assert.Equal(tc.wantGroups, gotGroups)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ go_library(
|
|||
visibility = ["//operators/constellation-node-operator:__subpackages__"],
|
||||
deps = [
|
||||
"//operators/constellation-node-operator/api/v1alpha1",
|
||||
"//operators/constellation-node-operator/internal/cloud/api",
|
||||
"//operators/constellation-node-operator/internal/poller",
|
||||
"@com_github_azure_azure_sdk_for_go_sdk_azcore//:azcore",
|
||||
"@com_github_azure_azure_sdk_for_go_sdk_azcore//runtime",
|
||||
|
|
@ -44,6 +45,7 @@ go_test(
|
|||
embed = [":client"],
|
||||
deps = [
|
||||
"//operators/constellation-node-operator/api/v1alpha1",
|
||||
"//operators/constellation-node-operator/internal/cloud/api",
|
||||
"//operators/constellation-node-operator/internal/poller",
|
||||
"@com_github_azure_azure_sdk_for_go_sdk_azcore//:azcore",
|
||||
"@com_github_azure_azure_sdk_for_go_sdk_azcore//runtime",
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ import (
|
|||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4"
|
||||
updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
|
||||
cspapi "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api"
|
||||
)
|
||||
|
||||
// GetScalingGroupImage returns the image URI of the scaling group.
|
||||
|
|
@ -79,34 +81,63 @@ func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error)
|
|||
}
|
||||
|
||||
// ListScalingGroups retrieves a list of scaling groups for the cluster.
|
||||
func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) {
|
||||
func (c *Client) ListScalingGroups(ctx context.Context, uid string) ([]cspapi.ScalingGroup, error) {
|
||||
results := []cspapi.ScalingGroup{}
|
||||
pager := c.scaleSetsAPI.NewListPager(c.config.ResourceGroup, nil)
|
||||
|
||||
for pager.More() {
|
||||
page, err := pager.NextPage(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("paging scale sets: %w", err)
|
||||
return nil, fmt.Errorf("paging scale sets: %w", err)
|
||||
}
|
||||
for _, scaleSet := range page.Value {
|
||||
if scaleSet == nil || scaleSet.ID == nil {
|
||||
continue
|
||||
}
|
||||
if scaleSet.Tags == nil || scaleSet.Tags["constellation-uid"] == nil || *scaleSet.Tags["constellation-uid"] != uid {
|
||||
if scaleSet.Tags == nil ||
|
||||
scaleSet.Tags["constellation-uid"] == nil ||
|
||||
*scaleSet.Tags["constellation-uid"] != uid ||
|
||||
scaleSet.Tags["constellation-role"] == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
role := updatev1alpha1.NodeRoleFromString(*scaleSet.Tags["constellation-role"])
|
||||
|
||||
name, err := c.GetScalingGroupName(*scaleSet.ID)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("getting scaling group name: %w", err)
|
||||
return nil, fmt.Errorf("getting scaling group name: %w", err)
|
||||
}
|
||||
switch *scaleSet.Tags["constellation-role"] {
|
||||
case "control-plane", "controlplane":
|
||||
controlPlaneGroupIDs = append(controlPlaneGroupIDs, *scaleSet.ID)
|
||||
case "worker":
|
||||
workerGroupIDs = append(workerGroupIDs, *scaleSet.ID)
|
||||
|
||||
var nodeGroupName string
|
||||
if scaleSet.Tags["constellation-node-group"] != nil {
|
||||
nodeGroupName = *scaleSet.Tags["constellation-node-group"]
|
||||
}
|
||||
// fallback for legacy clusters
|
||||
// TODO(malt3): remove this fallback once we can assume all clusters have the correct labels
|
||||
if nodeGroupName == "" {
|
||||
switch role {
|
||||
case updatev1alpha1.ControlPlaneRole:
|
||||
nodeGroupName = "control_plane_default"
|
||||
case updatev1alpha1.WorkerRole:
|
||||
nodeGroupName = "worker_default"
|
||||
}
|
||||
}
|
||||
|
||||
autoscalerGroupName, err := c.GetAutoscalingGroupName(*scaleSet.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting autoscaling group name: %w", err)
|
||||
}
|
||||
|
||||
results = append(results, cspapi.ScalingGroup{
|
||||
Name: name,
|
||||
NodeGroupName: nodeGroupName,
|
||||
GroupID: *scaleSet.ID,
|
||||
AutoscalingGroupName: autoscalerGroupName,
|
||||
Role: role,
|
||||
})
|
||||
}
|
||||
}
|
||||
return controlPlaneGroupIDs, workerGroupIDs, nil
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func imageReferenceFromImage(img string) *armcompute.ImageReference {
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import (
|
|||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4"
|
||||
cspapi "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -184,41 +185,67 @@ func TestGetScalingGroupName(t *testing.T) {
|
|||
|
||||
func TestListScalingGroups(t *testing.T) {
|
||||
testCases := map[string]struct {
|
||||
scaleSet armcompute.VirtualMachineScaleSet
|
||||
fetchPageErr error
|
||||
wantControlPlanes []string
|
||||
wantWorkers []string
|
||||
wantErr bool
|
||||
scaleSet armcompute.VirtualMachineScaleSet
|
||||
fetchPageErr error
|
||||
wantGroups []cspapi.ScalingGroup
|
||||
wantErr bool
|
||||
}{
|
||||
"listing control-plane works": {
|
||||
scaleSet: armcompute.VirtualMachineScaleSet{
|
||||
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-control-planes-uid"),
|
||||
Name: to.Ptr("constellation-scale-set-control-planes-uid"),
|
||||
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-control-planes-uid"),
|
||||
Tags: map[string]*string{
|
||||
"constellation-uid": to.Ptr("uid"),
|
||||
"constellation-role": to.Ptr("control-plane"),
|
||||
},
|
||||
},
|
||||
wantControlPlanes: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-control-planes-uid"},
|
||||
wantGroups: []cspapi.ScalingGroup{
|
||||
{
|
||||
Name: "constellation-scale-set-control-planes-uid",
|
||||
NodeGroupName: "control_plane_default",
|
||||
GroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-control-planes-uid",
|
||||
AutoscalingGroupName: "constellation-scale-set-control-planes-uid",
|
||||
Role: "ControlPlane",
|
||||
},
|
||||
},
|
||||
},
|
||||
"listing worker works": {
|
||||
scaleSet: armcompute.VirtualMachineScaleSet{
|
||||
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-workers-uid"),
|
||||
Name: to.Ptr("constellation-scale-set-workers-uid"),
|
||||
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-workers-uid"),
|
||||
Tags: map[string]*string{
|
||||
"constellation-uid": to.Ptr("uid"),
|
||||
"constellation-role": to.Ptr("worker"),
|
||||
},
|
||||
},
|
||||
wantWorkers: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-workers-uid"},
|
||||
wantGroups: []cspapi.ScalingGroup{
|
||||
{
|
||||
Name: "constellation-scale-set-workers-uid",
|
||||
NodeGroupName: "worker_default",
|
||||
GroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/constellation-scale-set-workers-uid",
|
||||
AutoscalingGroupName: "constellation-scale-set-workers-uid",
|
||||
Role: "Worker",
|
||||
},
|
||||
},
|
||||
},
|
||||
"listing is not dependent on resource name": {
|
||||
scaleSet: armcompute.VirtualMachineScaleSet{
|
||||
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/some-scale-set"),
|
||||
Name: to.Ptr("foo-bar"),
|
||||
ID: to.Ptr("/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/some-scale-set"),
|
||||
Tags: map[string]*string{
|
||||
"constellation-uid": to.Ptr("uid"),
|
||||
"constellation-role": to.Ptr("control-plane"),
|
||||
},
|
||||
},
|
||||
wantControlPlanes: []string{"/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/some-scale-set"},
|
||||
wantGroups: []cspapi.ScalingGroup{
|
||||
{
|
||||
Name: "some-scale-set",
|
||||
NodeGroupName: "control_plane_default",
|
||||
GroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/some-scale-set",
|
||||
AutoscalingGroupName: "some-scale-set",
|
||||
Role: "ControlPlane",
|
||||
},
|
||||
},
|
||||
},
|
||||
"listing other works": {
|
||||
scaleSet: armcompute.VirtualMachineScaleSet{
|
||||
|
|
@ -245,14 +272,13 @@ func TestListScalingGroups(t *testing.T) {
|
|||
},
|
||||
},
|
||||
}
|
||||
gotControlPlanes, gotWorkers, err := client.ListScalingGroups(context.Background(), "uid")
|
||||
gotGroups, err := client.ListScalingGroups(context.Background(), "uid")
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
return
|
||||
}
|
||||
require.NoError(err)
|
||||
assert.ElementsMatch(tc.wantControlPlanes, gotControlPlanes)
|
||||
assert.ElementsMatch(tc.wantWorkers, gotWorkers)
|
||||
assert.ElementsMatch(tc.wantGroups, gotGroups)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ go_library(
|
|||
visibility = ["//operators/constellation-node-operator:__subpackages__"],
|
||||
deps = [
|
||||
"//operators/constellation-node-operator/api/v1alpha1",
|
||||
"//operators/constellation-node-operator/internal/cloud/api",
|
||||
"//operators/constellation-node-operator/internal/constants",
|
||||
],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"fmt"
|
||||
|
||||
updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
|
||||
cspapi "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api"
|
||||
"github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/constants"
|
||||
)
|
||||
|
||||
|
|
@ -82,8 +83,23 @@ func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error)
|
|||
}
|
||||
|
||||
// ListScalingGroups retrieves a list of scaling groups for the cluster.
|
||||
func (c *Client) ListScalingGroups(_ context.Context, _ string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) {
|
||||
return []string{controlPlanesID}, []string{workersID}, nil
|
||||
func (c *Client) ListScalingGroups(_ context.Context, _ string) ([]cspapi.ScalingGroup, error) {
|
||||
return []cspapi.ScalingGroup{
|
||||
{
|
||||
Name: controlPlanesID,
|
||||
NodeGroupName: controlPlanesID,
|
||||
GroupID: controlPlanesID,
|
||||
AutoscalingGroupName: controlPlanesID,
|
||||
Role: updatev1alpha1.ControlPlaneRole,
|
||||
},
|
||||
{
|
||||
Name: workersID,
|
||||
NodeGroupName: workersID,
|
||||
GroupID: workersID,
|
||||
AutoscalingGroupName: workersID,
|
||||
Role: updatev1alpha1.WorkerRole,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler.
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ go_library(
|
|||
visibility = ["//operators/constellation-node-operator:__subpackages__"],
|
||||
deps = [
|
||||
"//operators/constellation-node-operator/api/v1alpha1",
|
||||
"//operators/constellation-node-operator/internal/cloud/api",
|
||||
"@com_github_googleapis_gax_go_v2//:gax-go",
|
||||
"@com_github_spf13_afero//:afero",
|
||||
"@com_google_cloud_go_compute//apiv1",
|
||||
|
|
@ -51,6 +52,7 @@ go_test(
|
|||
embed = [":client"],
|
||||
deps = [
|
||||
"//operators/constellation-node-operator/api/v1alpha1",
|
||||
"//operators/constellation-node-operator/internal/cloud/api",
|
||||
"@com_github_googleapis_gax_go_v2//:gax-go",
|
||||
"@com_github_spf13_afero//:afero",
|
||||
"@com_github_stretchr_testify//assert",
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ import (
|
|||
"strings"
|
||||
|
||||
"cloud.google.com/go/compute/apiv1/computepb"
|
||||
updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
|
||||
cspapi "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api"
|
||||
"google.golang.org/api/iterator"
|
||||
)
|
||||
|
||||
|
|
@ -106,7 +108,8 @@ func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error)
|
|||
}
|
||||
|
||||
// ListScalingGroups retrieves a list of scaling groups for the cluster.
|
||||
func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) {
|
||||
func (c *Client) ListScalingGroups(ctx context.Context, uid string) ([]cspapi.ScalingGroup, error) {
|
||||
results := []cspapi.ScalingGroup{}
|
||||
iter := c.instanceGroupManagersAPI.AggregatedList(ctx, &computepb.AggregatedListInstanceGroupManagersRequest{
|
||||
Project: c.projectID,
|
||||
})
|
||||
|
|
@ -115,7 +118,7 @@ func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlan
|
|||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("listing instance group managers: %w", err)
|
||||
return nil, fmt.Errorf("listing instance group managers: %w", err)
|
||||
}
|
||||
if instanceGroupManagerScopedListPair.Value == nil {
|
||||
continue
|
||||
|
|
@ -134,7 +137,7 @@ func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlan
|
|||
InstanceTemplate: templateURI[len(templateURI)-1],
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("getting instance template: %w", err)
|
||||
return nil, fmt.Errorf("getting instance template: %w", err)
|
||||
}
|
||||
if template.Properties == nil || template.Properties.Labels == nil {
|
||||
continue
|
||||
|
|
@ -145,18 +148,43 @@ func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlan
|
|||
|
||||
groupID, err := c.canonicalInstanceGroupID(ctx, *grpManager.SelfLink)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("normalizing instance group ID: %w", err)
|
||||
return nil, fmt.Errorf("normalizing instance group ID: %w", err)
|
||||
}
|
||||
|
||||
switch strings.ToLower(template.Properties.Labels["constellation-role"]) {
|
||||
case "control-plane", "controlplane":
|
||||
controlPlaneGroupIDs = append(controlPlaneGroupIDs, groupID)
|
||||
case "worker":
|
||||
workerGroupIDs = append(workerGroupIDs, groupID)
|
||||
role := updatev1alpha1.NodeRoleFromString(template.Properties.Labels["constellation-role"])
|
||||
|
||||
name, err := c.GetScalingGroupName(groupID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting scaling group name: %w", err)
|
||||
}
|
||||
|
||||
nodeGroupName := template.Properties.Labels["constellation-node-group"]
|
||||
// fallback for legacy clusters
|
||||
// TODO(malt3): remove this fallback once we can assume all clusters have the correct labels
|
||||
if nodeGroupName == "" {
|
||||
switch role {
|
||||
case updatev1alpha1.ControlPlaneRole:
|
||||
nodeGroupName = "control_plane_default"
|
||||
case updatev1alpha1.WorkerRole:
|
||||
nodeGroupName = "worker_default"
|
||||
}
|
||||
}
|
||||
|
||||
autoscalerGroupName, err := c.GetAutoscalingGroupName(groupID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting autoscaling group name: %w", err)
|
||||
}
|
||||
|
||||
results = append(results, cspapi.ScalingGroup{
|
||||
Name: name,
|
||||
NodeGroupName: nodeGroupName,
|
||||
GroupID: groupID,
|
||||
AutoscalingGroupName: autoscalerGroupName,
|
||||
Role: role,
|
||||
})
|
||||
}
|
||||
}
|
||||
return controlPlaneGroupIDs, workerGroupIDs, nil
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (c *Client) getScalingGroupTemplate(ctx context.Context, scalingGroupID string) (*computepb.InstanceTemplate, error) {
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
"testing"
|
||||
|
||||
"cloud.google.com/go/compute/apiv1/computepb"
|
||||
cspapi "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
|
@ -330,8 +331,7 @@ func TestListScalingGroups(t *testing.T) {
|
|||
templateLabels map[string]string
|
||||
listInstanceGroupManagersErr error
|
||||
templateGetErr error
|
||||
wantControlPlanes []string
|
||||
wantWorkers []string
|
||||
wantGroups []cspapi.ScalingGroup
|
||||
wantErr bool
|
||||
}{
|
||||
"list instance group managers fails": {
|
||||
|
|
@ -353,8 +353,14 @@ func TestListScalingGroups(t *testing.T) {
|
|||
"constellation-uid": "uid",
|
||||
"constellation-role": "control-plane",
|
||||
},
|
||||
wantControlPlanes: []string{
|
||||
"projects/project/zones/zone/instanceGroupManagers/test-control-plane-uid",
|
||||
wantGroups: []cspapi.ScalingGroup{
|
||||
{
|
||||
Name: "test-control-plane-uid",
|
||||
NodeGroupName: "control_plane_default",
|
||||
GroupID: "projects/project/zones/zone/instanceGroupManagers/test-control-plane-uid",
|
||||
AutoscalingGroupName: "https://www.googleapis.com/compute/v1/projects/project/zones/zone/instanceGroups/test-control-plane-uid",
|
||||
Role: "ControlPlane",
|
||||
},
|
||||
},
|
||||
},
|
||||
"list instance group managers for worker": {
|
||||
|
|
@ -365,8 +371,33 @@ func TestListScalingGroups(t *testing.T) {
|
|||
"constellation-uid": "uid",
|
||||
"constellation-role": "worker",
|
||||
},
|
||||
wantWorkers: []string{
|
||||
"projects/project/zones/zone/instanceGroupManagers/test-worker-uid",
|
||||
wantGroups: []cspapi.ScalingGroup{
|
||||
{
|
||||
Name: "test-worker-uid",
|
||||
NodeGroupName: "worker_default",
|
||||
GroupID: "projects/project/zones/zone/instanceGroupManagers/test-worker-uid",
|
||||
AutoscalingGroupName: "https://www.googleapis.com/compute/v1/projects/project/zones/zone/instanceGroups/test-worker-uid",
|
||||
Role: "Worker",
|
||||
},
|
||||
},
|
||||
},
|
||||
"list instance group managers with custom group name": {
|
||||
name: proto.String("test-worker-uid"),
|
||||
groupID: proto.String("projects/project/zones/zone/instanceGroupManagers/test-worker-uid"),
|
||||
templateRef: proto.String("projects/project/global/instanceTemplates/test-control-plane-uid"),
|
||||
templateLabels: map[string]string{
|
||||
"constellation-uid": "uid",
|
||||
"constellation-role": "worker",
|
||||
"constellation-node-group": "custom-group-name",
|
||||
},
|
||||
wantGroups: []cspapi.ScalingGroup{
|
||||
{
|
||||
Name: "test-worker-uid",
|
||||
NodeGroupName: "custom-group-name",
|
||||
GroupID: "projects/project/zones/zone/instanceGroupManagers/test-worker-uid",
|
||||
AutoscalingGroupName: "https://www.googleapis.com/compute/v1/projects/project/zones/zone/instanceGroups/test-worker-uid",
|
||||
Role: "Worker",
|
||||
},
|
||||
},
|
||||
},
|
||||
"listing instance group managers is not dependant on resource name": {
|
||||
|
|
@ -377,8 +408,14 @@ func TestListScalingGroups(t *testing.T) {
|
|||
"constellation-uid": "uid",
|
||||
"constellation-role": "control-plane",
|
||||
},
|
||||
wantControlPlanes: []string{
|
||||
"projects/project/zones/zone/instanceGroupManagers/some-instance-group-manager",
|
||||
wantGroups: []cspapi.ScalingGroup{
|
||||
{
|
||||
Name: "some-instance-group-manager",
|
||||
NodeGroupName: "control_plane_default",
|
||||
GroupID: "projects/project/zones/zone/instanceGroupManagers/some-instance-group-manager",
|
||||
AutoscalingGroupName: "https://www.googleapis.com/compute/v1/projects/project/zones/zone/instanceGroups/some-instance-group-manager",
|
||||
Role: "ControlPlane",
|
||||
},
|
||||
},
|
||||
},
|
||||
"unrelated instance group manager": {
|
||||
|
|
@ -415,14 +452,13 @@ func TestListScalingGroups(t *testing.T) {
|
|||
getErr: tc.templateGetErr,
|
||||
},
|
||||
}
|
||||
gotControlPlanes, gotWorkers, err := client.ListScalingGroups(context.Background(), "uid")
|
||||
gotGroups, err := client.ListScalingGroups(context.Background(), "uid")
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
return
|
||||
}
|
||||
require.NoError(err)
|
||||
assert.ElementsMatch(tc.wantControlPlanes, gotControlPlanes)
|
||||
assert.ElementsMatch(tc.wantWorkers, gotWorkers)
|
||||
assert.ElementsMatch(tc.wantGroups, gotGroups)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue