Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-11-13 00:50:38 -05:00
Code Issues Projects Releases Packages Wiki Activity

Let operator manage autoscaling of node groups

Browse source
This commit is contained in:
katexochen 2022-09-15 16:48:32 +02:00 • committed by Paul Meyer
parent 67d9be38d7
commit e301f575df
18 changed files with 499 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

12
operators/constellation-node-operator/internal/azure/client/autoscaler.go Normal file
View file

@ -0,0 +1,12 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
// AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler.
func (c *Client) AutoscalingCloudProvider() string {
return "azure"
}

13
operators/constellation-node-operator/internal/azure/client/scalinggroup.go
View file

@ -63,13 +63,20 @@ func (c *Client) SetScalingGroupImage(ctx context.Context, scalingGroupID, image
return nil
}
// GetScalingGroupName retrieves the name of a scaling group.
func (c *Client) GetScalingGroupName(ctx context.Context, scalingGroupID string) (string, error) {
// GetScalingGroupName retrieves the name of a scaling group, as expected by Kubernetes.
// This keeps the casing of the original name, but Kubernetes requires the name to be lowercase,
// so use strings.ToLower() on the result if using the name in a Kubernetes context.
func (c *Client) GetScalingGroupName(scalingGroupID string) (string, error) {
_, _, scaleSet, err := splitVMSSID(scalingGroupID)
if err != nil {
return "", fmt.Errorf("getting scaling group name: %w", err)
}
return strings.ToLower(scaleSet), nil
return scaleSet, nil
}
// GetScalingGroupName retrieves the name of a scaling group as needed by the cluster-autoscaler.
func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error) {
return c.GetScalingGroupName(scalingGroupID)
}
// ListScalingGroups retrieves a list of scaling groups for the cluster.

6
operators/constellation-node-operator/internal/azure/client/scalinggroup_test.go
View file

@ -155,9 +155,9 @@ func TestGetScalingGroupName(t *testing.T) {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scale-set-name",
wantName: "scale-set-name",
},
"uppercase name is lowercased": {
"uppercase name isn't lowercased": {
scalingGroupID: "/subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/SCALE-SET-NAME",
wantName: "scale-set-name",
wantName: "SCALE-SET-NAME",
},
"splitting scalingGroupID fails": {
scalingGroupID: "invalid",
@ -171,7 +171,7 @@ func TestGetScalingGroupName(t *testing.T) {
require := require.New(t)
client := Client{}
gotName, err := client.GetScalingGroupName(context.Background(), tc.scalingGroupID)
gotName, err := client.GetScalingGroupName(tc.scalingGroupID)
if tc.wantErr {
assert.Error(err)
return

47
operators/constellation-node-operator/internal/deploy/deploy.go
View file

@ -11,6 +11,7 @@ import (
"context"
"errors"
"fmt"
"strings"
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/api/v1alpha1"
"github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/constants"
@ -32,7 +33,7 @@ func InitialResources(ctx context.Context, k8sClient client.Writer, scalingGroup
return errors.New("determining initial node image: no worker scaling group found")
}
if err := createAutoscalingStrategy(ctx, k8sClient); err != nil {
if err := createAutoscalingStrategy(ctx, k8sClient, scalingGroupGetter.AutoscalingCloudProvider()); err != nil {
return fmt.Errorf("creating initial autoscaling strategy: %w", err)
}
imageReference, err := scalingGroupGetter.GetScalingGroupImage(ctx, controlPlaneGroupIDs[0])
@ -43,20 +44,28 @@ func InitialResources(ctx context.Context, k8sClient client.Writer, scalingGroup
return fmt.Errorf("creating initial node image %q: %w", imageReference, err)
}
for _, groupID := range controlPlaneGroupIDs {
groupName, err := scalingGroupGetter.GetScalingGroupName(ctx, groupID)
groupName, err := scalingGroupGetter.GetScalingGroupName(groupID)
if err != nil {
return fmt.Errorf("determining scaling group name of %q: %w", groupID, err)
}
if err := createScalingGroup(ctx, k8sClient, groupID, groupName, false); err != nil {
autoscalingGroupName, err := scalingGroupGetter.GetAutoscalingGroupName(groupID)
if err != nil {
return fmt.Errorf("determining autoscaling group name of %q: %w", groupID, err)
}
if err := createScalingGroup(ctx, k8sClient, groupID, groupName, autoscalingGroupName, updatev1alpha1.ControlPlaneRole); err != nil {
return fmt.Errorf("creating initial control plane scaling group: %w", err)
}
}
for _, groupID := range workerGroupIDs {
groupName, err := scalingGroupGetter.GetScalingGroupName(ctx, groupID)
groupName, err := scalingGroupGetter.GetScalingGroupName(groupID)
if err != nil {
return fmt.Errorf("determining scaling group name of %q: %w", groupID, err)
}
if err := createScalingGroup(ctx, k8sClient, groupID, groupName, true); err != nil {
autoscalingGroupName, err := scalingGroupGetter.GetAutoscalingGroupName(groupID)
if err != nil {
return fmt.Errorf("determining autoscaling group name of %q: %w", groupID, err)
}
if err := createScalingGroup(ctx, k8sClient, groupID, groupName, autoscalingGroupName, updatev1alpha1.WorkerRole); err != nil {
return fmt.Errorf("creating initial worker scaling group: %w", err)
}
}
@ -64,7 +73,7 @@ func InitialResources(ctx context.Context, k8sClient client.Writer, scalingGroup
}
// createAutoscalingStrategy creates the autoscaling strategy resource if it does not exist yet.
func createAutoscalingStrategy(ctx context.Context, k8sClient client.Writer) error {
func createAutoscalingStrategy(ctx context.Context, k8sClient client.Writer, provider string) error {
err := k8sClient.Create(ctx, &updatev1alpha1.AutoscalingStrategy{
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "AutoscalingStrategy"},
ObjectMeta: metav1.ObjectMeta{
@ -74,6 +83,13 @@ func createAutoscalingStrategy(ctx context.Context, k8sClient client.Writer) err
Enabled: true,
DeploymentName: "constellation-cluster-autoscaler",
DeploymentNamespace: "kube-system",
AutoscalerExtraArgs: map[string]string{
"cloud-provider": provider,
"logtostderr": "true",
"stderrthreshold": "info",
"v": "2",
"namespace": "kube-system",
},
},
})
if k8sErrors.IsAlreadyExists(err) {
@ -100,16 +116,19 @@ func createNodeImage(ctx context.Context, k8sClient client.Writer, imageReferenc
}
// createScalingGroup creates an initial scaling group resource if it does not exist yet.
func createScalingGroup(ctx context.Context, k8sClient client.Writer, groupID, groupName string, autoscaling bool) error {
func createScalingGroup(ctx context.Context, k8sClient client.Writer, groupID, groupName, autoscalingGroupName string, role updatev1alpha1.NodeRole) error {
err := k8sClient.Create(ctx, &updatev1alpha1.ScalingGroup{
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "ScalingGroup"},
ObjectMeta: metav1.ObjectMeta{
Name: groupName,
Name: strings.ToLower(groupName),
},
Spec: updatev1alpha1.ScalingGroupSpec{
NodeImage: constants.NodeImageResourceName,
GroupID: groupID,
Autoscaling: autoscaling,
NodeImage: constants.NodeImageResourceName,
GroupID: groupID,
AutoscalerGroupName: autoscalingGroupName,
Min: 1,
Max: 10,
Role: role,
},
})
if k8sErrors.IsAlreadyExists(err) {
@ -122,7 +141,11 @@ type scalingGroupGetter interface {
// GetScalingGroupImage retrieves the image currently used by a scaling group.
GetScalingGroupImage(ctx context.Context, scalingGroupID string) (string, error)
// GetScalingGroupName retrieves the name of a scaling group.
GetScalingGroupName(ctx context.Context, scalingGroupID string) (string, error)
GetScalingGroupName(scalingGroupID string) (string, error)
// GetScalingGroupName retrieves the name of a scaling group as needed by the cluster-autoscaler.
GetAutoscalingGroupName(scalingGroupID string) (string, error)
// ListScalingGroups retrieves a list of scaling groups for the cluster.
ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error)
// AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler.
AutoscalingCloudProvider() string
}

46
operators/constellation-node-operator/internal/deploy/deploy_test.go
View file

@ -114,6 +114,13 @@ func TestCreateAutoscalingStrategy(t *testing.T) {
Enabled: true,
DeploymentName: "constellation-cluster-autoscaler",
DeploymentNamespace: "kube-system",
AutoscalerExtraArgs: map[string]string{
"cloud-provider": "stub",
"logtostderr": "true",
"stderrthreshold": "info",
"v": "2",
"namespace": "kube-system",
},
},
},
},
@ -132,6 +139,13 @@ func TestCreateAutoscalingStrategy(t *testing.T) {
Enabled: true,
DeploymentName: "constellation-cluster-autoscaler",
DeploymentNamespace: "kube-system",
AutoscalerExtraArgs: map[string]string{
"cloud-provider": "stub",
"logtostderr": "true",
"stderrthreshold": "info",
"v": "2",
"namespace": "kube-system",
},
},
},
},
@ -143,7 +157,7 @@ func TestCreateAutoscalingStrategy(t *testing.T) {
require := require.New(t)
k8sClient := &stubK8sClient{createErr: tc.createErr}
err := createAutoscalingStrategy(context.Background(), k8sClient)
err := createAutoscalingStrategy(context.Background(), k8sClient, "stub")
if tc.wantErr {
assert.Error(err)
return
@ -221,9 +235,12 @@ func TestCreateScalingGroup(t *testing.T) {
Name: "group-name",
},
Spec: updatev1alpha1.ScalingGroupSpec{
NodeImage: constants.NodeImageResourceName,
GroupID: "group-id",
Autoscaling: true,
NodeImage: constants.NodeImageResourceName,
GroupID: "group-id",
AutoscalerGroupName: "group-Name",
Min: 1,
Max: 10,
Role: updatev1alpha1.WorkerRole,
},
},
},
@ -239,9 +256,12 @@ func TestCreateScalingGroup(t *testing.T) {
Name: "group-name",
},
Spec: updatev1alpha1.ScalingGroupSpec{
NodeImage: constants.NodeImageResourceName,
GroupID: "group-id",
Autoscaling: true,
NodeImage: constants.NodeImageResourceName,
GroupID: "group-id",
AutoscalerGroupName: "group-Name",
Min: 1,
Max: 10,
Role: updatev1alpha1.WorkerRole,
},
},
},
@ -253,7 +273,7 @@ func TestCreateScalingGroup(t *testing.T) {
require := require.New(t)
k8sClient := &stubK8sClient{createErr: tc.createErr}
err := createScalingGroup(context.Background(), k8sClient, "group-id", "group-name", true)
err := createScalingGroup(context.Background(), k8sClient, "group-id", "group-Name", "group-Name", updatev1alpha1.WorkerRole)
if tc.wantErr {
assert.Error(err)
return
@ -300,7 +320,11 @@ func (g *stubScalingGroupGetter) GetScalingGroupImage(ctx context.Context, scali
return g.store[scalingGroupID].image, g.imageErr
}
func (g *stubScalingGroupGetter) GetScalingGroupName(ctx context.Context, scalingGroupID string) (string, error) {
func (g *stubScalingGroupGetter) GetScalingGroupName(scalingGroupID string) (string, error) {
return g.store[scalingGroupID].name, g.nameErr
}
func (g *stubScalingGroupGetter) GetAutoscalingGroupName(scalingGroupID string) (string, error) {
return g.store[scalingGroupID].name, g.nameErr
}
@ -315,6 +339,10 @@ func (g *stubScalingGroupGetter) ListScalingGroups(ctx context.Context, uid stri
return controlPlaneGroupIDs, workerGroupIDs, g.listErr
}
func (g *stubScalingGroupGetter) AutoscalingCloudProvider() string {
return "stub"
}
type scalingGroupStoreItem struct {
groupID string
name string

12
operators/constellation-node-operator/internal/gcp/client/autocaler.go Normal file
View file

@ -0,0 +1,12 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package client
// AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler.
func (c *Client) AutoscalingCloudProvider() string {
return "gce"
}

9
operators/constellation-node-operator/internal/gcp/client/disks.go
View file

@ -40,3 +40,12 @@ func uriNormalize(imageURI string) string {
}
return matches[1]
}
// ensureURIPrefixed ensures that a compute API URI is prefixed with the optional URI prefix.
func ensureURIPrefixed(uri string) string {
matches := computeAPIBase.FindStringSubmatch(uri)
if len(matches) == 2 {
return uri
}
return "https://www.googleapis.com/compute/v1/" + uri
}

16
operators/constellation-node-operator/internal/gcp/client/scalinggroup.go
View file

@ -10,7 +10,6 @@ import (
"context"
"errors"
"fmt"
"strings"
"google.golang.org/api/iterator"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
@ -87,12 +86,23 @@ func (c *Client) SetScalingGroupImage(ctx context.Context, scalingGroupID, image
}
// GetScalingGroupName retrieves the name of a scaling group.
func (c *Client) GetScalingGroupName(ctx context.Context, scalingGroupID string) (string, error) {
// This keeps the casing of the original name, but Kubernetes requires the name to be lowercase,
// so use strings.ToLower() on the result if using the name in a Kubernetes context.
func (c *Client) GetScalingGroupName(scalingGroupID string) (string, error) {
_, _, instanceGroupName, err := splitInstanceGroupID(scalingGroupID)
if err != nil {
return "", fmt.Errorf("getting scaling group name: %w", err)
}
return strings.ToLower(instanceGroupName), nil
return instanceGroupName, nil
}
// GetScalingGroupName retrieves the name of a scaling group as needed by the cluster-autoscaler.
func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error) {
project, zone, instanceGroupName, err := splitInstanceGroupID(scalingGroupID)
if err != nil {
return "", fmt.Errorf("getting autoscaling scaling group name: %w", err)
}
return ensureURIPrefixed(fmt.Sprintf("projects/%s/zones/%s/instanceGroups/%s", project, zone, instanceGroupName)), nil
}
// ListScalingGroups retrieves a list of scaling groups for the cluster.

2
operators/constellation-node-operator/internal/gcp/client/scalinggroup_test.go
View file

@ -311,7 +311,7 @@ func TestGetScalingGroupName(t *testing.T) {
require := require.New(t)
client := Client{}
gotName, err := client.GetScalingGroupName(context.Background(), tc.scalingGroupID)
gotName, err := client.GetScalingGroupName(tc.scalingGroupID)
if tc.wantErr {
assert.Error(err)
return