mirror of
https://github.com/edgelesssys/constellation.git
synced 2025-05-14 12:12:25 -04:00
operators: infrastructure autodiscovery (#1958)
* helm: configure GCP cloud controller manager to search in all zones of a region
See also: d716fdd452/providers/gce/gce.go (L376-L380)
* operators: add nodeGroupName to ScalingGroup CRD
NodeGroupName is the human friendly name of the node group that will be exposed to customers via the Constellation config in the future.
* operators: support simple executor / scheduler to reconcile on non-k8s resources
* operators: add new return type for ListScalingGroups to support arbitrary node groups
* operators: ListScalingGroups should return additionally created node groups on AWS
* operators: ListScalingGroups should return additionally created node groups on Azure
* operators: ListScalingGroups should return additionally created node groups on GCP
* operators: ListScalingGroups should return additionally created node groups on unsupported CSPs
* operators: implement external scaling group reconciler
This controller scans the cloud provider infrastructure and changes k8s resources accordingly.
It creates ScaleSet resources when new node groups are created and deletes them if the node groups are removed.
* operators: no longer create scale sets when the operator starts
In the future, scale sets are created dynamically.
* operators: watch for node join/leave events using a controller
* operators: deploy new controllers
* docs: update auto scaling documentation with support for node groups
This commit is contained in:
Malte Poll
Adrian Stobbe
parent
10a540c290
commit
388ff011a3
36 changed files with 1836 additions and 232 deletions
|
|
@ -12,6 +12,7 @@ go_library(
|
|||
deps = [
|
||||
"//internal/constants",
|
||||
"//operators/constellation-node-operator/api/v1alpha1",
|
||||
"//operators/constellation-node-operator/internal/cloud/api",
|
||||
"//operators/constellation-node-operator/internal/constants",
|
||||
"@com_github_spf13_afero//:afero",
|
||||
"@io_k8s_api//core/v1:core",
|
||||
|
|
@ -33,6 +34,7 @@ go_test(
|
|||
deps = [
|
||||
"//internal/constants",
|
||||
"//operators/constellation-node-operator/api/v1alpha1",
|
||||
"//operators/constellation-node-operator/internal/cloud/api",
|
||||
"//operators/constellation-node-operator/internal/constants",
|
||||
"@com_github_spf13_afero//:afero",
|
||||
"@com_github_stretchr_testify//assert",
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import (
|
|||
|
||||
mainconstants "github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
|
||||
cspapi "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api"
|
||||
"github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/constants"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
|
|
@ -33,21 +34,18 @@ func InitialResources(ctx context.Context, k8sClient client.Client, imageInfo im
|
|||
}
|
||||
logr.Info("cleaned up placeholders")
|
||||
|
||||
controlPlaneGroupIDs, workerGroupIDs, err := scalingGroupGetter.ListScalingGroups(ctx, uid)
|
||||
scalingGroups, err := scalingGroupGetter.ListScalingGroups(ctx, uid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing scaling groups: %w", err)
|
||||
}
|
||||
if len(controlPlaneGroupIDs) == 0 {
|
||||
return errors.New("determining initial node image: no control plane scaling group found")
|
||||
}
|
||||
if len(workerGroupIDs) == 0 {
|
||||
return errors.New("determining initial node image: no worker scaling group found")
|
||||
if len(scalingGroups) == 0 {
|
||||
return errors.New("determining initial node image: no scaling group found")
|
||||
}
|
||||
|
||||
if err := createAutoscalingStrategy(ctx, k8sClient, scalingGroupGetter.AutoscalingCloudProvider()); err != nil {
|
||||
return fmt.Errorf("creating initial autoscaling strategy: %w", err)
|
||||
}
|
||||
imageReference, err := scalingGroupGetter.GetScalingGroupImage(ctx, controlPlaneGroupIDs[0])
|
||||
imageReference, err := scalingGroupGetter.GetScalingGroupImage(ctx, scalingGroups[0].GroupID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("determining initial node image: %w", err)
|
||||
}
|
||||
|
|
@ -62,34 +60,6 @@ func InitialResources(ctx context.Context, k8sClient client.Client, imageInfo im
|
|||
if err := createNodeVersion(ctx, k8sClient, imageReference, imageVersion); err != nil {
|
||||
return fmt.Errorf("creating initial node version %q: %w", imageReference, err)
|
||||
}
|
||||
for _, groupID := range controlPlaneGroupIDs {
|
||||
groupName, err := scalingGroupGetter.GetScalingGroupName(groupID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("determining scaling group name of %q: %w", groupID, err)
|
||||
}
|
||||
autoscalingGroupName, err := scalingGroupGetter.GetAutoscalingGroupName(groupID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("determining autoscaling group name of %q: %w", groupID, err)
|
||||
}
|
||||
newScalingGroupConfig := newScalingGroupConfig{k8sClient, groupID, groupName, autoscalingGroupName, updatev1alpha1.ControlPlaneRole}
|
||||
if err := createScalingGroup(ctx, newScalingGroupConfig); err != nil {
|
||||
return fmt.Errorf("creating initial control plane scaling group: %w", err)
|
||||
}
|
||||
}
|
||||
for _, groupID := range workerGroupIDs {
|
||||
groupName, err := scalingGroupGetter.GetScalingGroupName(groupID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("determining scaling group name of %q: %w", groupID, err)
|
||||
}
|
||||
autoscalingGroupName, err := scalingGroupGetter.GetAutoscalingGroupName(groupID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("determining autoscaling group name of %q: %w", groupID, err)
|
||||
}
|
||||
newScalingGroupConfig := newScalingGroupConfig{k8sClient, groupID, groupName, autoscalingGroupName, updatev1alpha1.WorkerRole}
|
||||
if err := createScalingGroup(ctx, newScalingGroupConfig); err != nil {
|
||||
return fmt.Errorf("creating initial worker scaling group: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -252,28 +222,6 @@ func findLatestK8sComponentsConfigMap(ctx context.Context, k8sClient client.Clie
|
|||
return latestConfigMap, nil
|
||||
}
|
||||
|
||||
// createScalingGroup creates an initial scaling group resource if it does not exist yet.
|
||||
func createScalingGroup(ctx context.Context, config newScalingGroupConfig) error {
|
||||
err := config.k8sClient.Create(ctx, &updatev1alpha1.ScalingGroup{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "ScalingGroup"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: strings.ToLower(config.groupName),
|
||||
},
|
||||
Spec: updatev1alpha1.ScalingGroupSpec{
|
||||
NodeVersion: mainconstants.NodeVersionResourceName,
|
||||
GroupID: config.groupID,
|
||||
AutoscalerGroupName: config.autoscalingGroupName,
|
||||
Min: 1,
|
||||
Max: 10,
|
||||
Role: config.role,
|
||||
},
|
||||
})
|
||||
if k8sErrors.IsAlreadyExists(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type imageInfoGetter interface {
|
||||
ImageVersion() (string, error)
|
||||
}
|
||||
|
|
@ -286,15 +234,7 @@ type scalingGroupGetter interface {
|
|||
// GetScalingGroupName retrieves the name of a scaling group as needed by the cluster-autoscaler.
|
||||
GetAutoscalingGroupName(scalingGroupID string) (string, error)
|
||||
// ListScalingGroups retrieves a list of scaling groups for the cluster.
|
||||
ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error)
|
||||
ListScalingGroups(ctx context.Context, uid string) ([]cspapi.ScalingGroup, error)
|
||||
// AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler.
|
||||
AutoscalingCloudProvider() string
|
||||
}
|
||||
|
||||
type newScalingGroupConfig struct {
|
||||
k8sClient client.Writer
|
||||
groupID string
|
||||
groupName string
|
||||
autoscalingGroupName string
|
||||
role updatev1alpha1.NodeRole
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import (
|
|||
|
||||
mainconstants "github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
|
||||
cspapi "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api"
|
||||
"github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/constants"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
|
@ -41,18 +42,10 @@ func TestInitialResources(t *testing.T) {
|
|||
{groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true},
|
||||
{groupID: "worker", image: "image-1", name: "worker"},
|
||||
},
|
||||
wantResources: 4,
|
||||
wantResources: 2,
|
||||
},
|
||||
"missing control planes": {
|
||||
items: []scalingGroupStoreItem{
|
||||
{groupID: "worker", image: "image-1", name: "worker"},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"missing workers": {
|
||||
items: []scalingGroupStoreItem{
|
||||
{groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true},
|
||||
},
|
||||
"missing groups": {
|
||||
items: []scalingGroupStoreItem{},
|
||||
wantErr: true,
|
||||
},
|
||||
"listing groups fails": {
|
||||
|
|
@ -75,14 +68,6 @@ func TestInitialResources(t *testing.T) {
|
|||
imageErr: errors.New("getting image failed"),
|
||||
wantErr: true,
|
||||
},
|
||||
"getting name fails": {
|
||||
items: []scalingGroupStoreItem{
|
||||
{groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true},
|
||||
{groupID: "worker", image: "image-1", name: "worker"},
|
||||
},
|
||||
nameErr: errors.New("getting name failed"),
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range testCases {
|
||||
|
|
@ -273,70 +258,6 @@ func TestCreateNodeVersion(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestCreateScalingGroup(t *testing.T) {
|
||||
testCases := map[string]struct {
|
||||
createErr error
|
||||
wantScalingGroup *updatev1alpha1.ScalingGroup
|
||||
wantErr bool
|
||||
}{
|
||||
"create works": {
|
||||
wantScalingGroup: &updatev1alpha1.ScalingGroup{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "ScalingGroup"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "group-name",
|
||||
},
|
||||
Spec: updatev1alpha1.ScalingGroupSpec{
|
||||
NodeVersion: mainconstants.NodeVersionResourceName,
|
||||
GroupID: "group-id",
|
||||
AutoscalerGroupName: "group-Name",
|
||||
Min: 1,
|
||||
Max: 10,
|
||||
Role: updatev1alpha1.WorkerRole,
|
||||
},
|
||||
},
|
||||
},
|
||||
"create fails": {
|
||||
createErr: errors.New("create failed"),
|
||||
wantErr: true,
|
||||
},
|
||||
"image exists": {
|
||||
createErr: k8sErrors.NewAlreadyExists(schema.GroupResource{}, constants.AutoscalingStrategyResourceName),
|
||||
wantScalingGroup: &updatev1alpha1.ScalingGroup{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "ScalingGroup"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "group-name",
|
||||
},
|
||||
Spec: updatev1alpha1.ScalingGroupSpec{
|
||||
NodeVersion: mainconstants.NodeVersionResourceName,
|
||||
GroupID: "group-id",
|
||||
AutoscalerGroupName: "group-Name",
|
||||
Min: 1,
|
||||
Max: 10,
|
||||
Role: updatev1alpha1.WorkerRole,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range testCases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
require := require.New(t)
|
||||
|
||||
k8sClient := &fakeK8sClient{createErr: tc.createErr}
|
||||
newScalingGroupConfig := newScalingGroupConfig{k8sClient, "group-id", "group-Name", "group-Name", updatev1alpha1.WorkerRole}
|
||||
err := createScalingGroup(context.Background(), newScalingGroupConfig)
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
return
|
||||
}
|
||||
require.NoError(err)
|
||||
assert.Len(k8sClient.createdObjects, 1)
|
||||
assert.Equal(tc.wantScalingGroup, k8sClient.createdObjects[0])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type fakeK8sClient struct {
|
||||
createdObjects []client.Object
|
||||
createErr error
|
||||
|
|
@ -437,15 +358,24 @@ func (g *stubScalingGroupGetter) GetAutoscalingGroupName(scalingGroupID string)
|
|||
return g.store[scalingGroupID].name, g.nameErr
|
||||
}
|
||||
|
||||
func (g *stubScalingGroupGetter) ListScalingGroups(_ context.Context, _ string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) {
|
||||
func (g *stubScalingGroupGetter) ListScalingGroups(_ context.Context, _ string) ([]cspapi.ScalingGroup, error) {
|
||||
var scalingGroups []cspapi.ScalingGroup
|
||||
|
||||
for _, item := range g.store {
|
||||
if item.isControlPlane {
|
||||
controlPlaneGroupIDs = append(controlPlaneGroupIDs, item.groupID)
|
||||
} else {
|
||||
workerGroupIDs = append(workerGroupIDs, item.groupID)
|
||||
}
|
||||
scalingGroups = append(scalingGroups, cspapi.ScalingGroup{
|
||||
Name: item.name,
|
||||
NodeGroupName: item.nodeGroupName,
|
||||
GroupID: item.groupID,
|
||||
AutoscalingGroupName: item.autoscalingGroupName,
|
||||
Role: func() updatev1alpha1.NodeRole {
|
||||
if item.isControlPlane {
|
||||
return updatev1alpha1.ControlPlaneRole
|
||||
}
|
||||
return updatev1alpha1.WorkerRole
|
||||
}(),
|
||||
})
|
||||
}
|
||||
return controlPlaneGroupIDs, workerGroupIDs, g.listErr
|
||||
return scalingGroups, g.listErr
|
||||
}
|
||||
|
||||
func (g *stubScalingGroupGetter) AutoscalingCloudProvider() string {
|
||||
|
|
@ -453,8 +383,10 @@ func (g *stubScalingGroupGetter) AutoscalingCloudProvider() string {
|
|||
}
|
||||
|
||||
type scalingGroupStoreItem struct {
|
||||
groupID string
|
||||
name string
|
||||
image string
|
||||
isControlPlane bool
|
||||
name string
|
||||
groupID string
|
||||
autoscalingGroupName string
|
||||
nodeGroupName string
|
||||
image string
|
||||
isControlPlane bool
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue