mirror of
https://github.com/edgelesssys/constellation.git
synced 2025-05-02 06:16:08 -04:00
[node operator] self-initialize resources
Signed-off-by: Malte Poll <mp@edgeless.systems>
This commit is contained in:
Malte Poll
Malte Poll
parent
1cee319174
commit
51cf638361
27 changed files with 1021 additions and 26 deletions
122
operators/constellation-node-operator/internal/deploy/deploy.go
Normal file
122
operators/constellation-node-operator/internal/deploy/deploy.go
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
// Package deploy provides functions to deploy initial resources for the node operator.
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/api/v1alpha1"
|
||||
"github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/constants"
|
||||
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
|
||||
// InitialResources creates the initial resources for the node operator.
|
||||
func InitialResources(ctx context.Context, k8sClient client.Writer, scalingGroupGetter scalingGroupGetter, uid string) error {
|
||||
controlPlaneGroupIDs, workerGroupIDs, err := scalingGroupGetter.ListScalingGroups(ctx, uid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing scaling groups: %w", err)
|
||||
}
|
||||
if len(controlPlaneGroupIDs) == 0 {
|
||||
return errors.New("determining initial node image: no control plane scaling group found")
|
||||
}
|
||||
if len(workerGroupIDs) == 0 {
|
||||
return errors.New("determining initial node image: no worker scaling group found")
|
||||
}
|
||||
|
||||
if err := createAutoscalingStrategy(ctx, k8sClient); err != nil {
|
||||
return fmt.Errorf("creating initial autoscaling strategy: %w", err)
|
||||
}
|
||||
imageReference, err := scalingGroupGetter.GetScalingGroupImage(ctx, controlPlaneGroupIDs[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("determining initial node image: %w", err)
|
||||
}
|
||||
if err := createNodeImage(ctx, k8sClient, imageReference); err != nil {
|
||||
return fmt.Errorf("creating initial node image %q: %w", imageReference, err)
|
||||
}
|
||||
for _, groupID := range controlPlaneGroupIDs {
|
||||
groupName, err := scalingGroupGetter.GetScalingGroupName(ctx, groupID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("determining scaling group name of %q: %w", groupID, err)
|
||||
}
|
||||
if err := createScalingGroup(ctx, k8sClient, groupID, groupName, false); err != nil {
|
||||
return fmt.Errorf("creating initial control plane scaling group: %w", err)
|
||||
}
|
||||
}
|
||||
for _, groupID := range workerGroupIDs {
|
||||
groupName, err := scalingGroupGetter.GetScalingGroupName(ctx, groupID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("determining scaling group name of %q: %w", groupID, err)
|
||||
}
|
||||
if err := createScalingGroup(ctx, k8sClient, groupID, groupName, true); err != nil {
|
||||
return fmt.Errorf("creating initial worker scaling group: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// createAutoscalingStrategy creates the autoscaling strategy resource if it does not exist yet.
|
||||
func createAutoscalingStrategy(ctx context.Context, k8sClient client.Writer) error {
|
||||
err := k8sClient.Create(ctx, &updatev1alpha1.AutoscalingStrategy{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "AutoscalingStrategy"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: constants.AutoscalingStrategyResourceName,
|
||||
},
|
||||
Spec: updatev1alpha1.AutoscalingStrategySpec{
|
||||
Enabled: true,
|
||||
DeploymentName: "constellation-cluster-autoscaler",
|
||||
DeploymentNamespace: "kube-system",
|
||||
},
|
||||
})
|
||||
if k8sErrors.IsAlreadyExists(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// createNodeImage creates the initial nodeimage resource if it does not exist yet.
|
||||
func createNodeImage(ctx context.Context, k8sClient client.Writer, imageReference string) error {
|
||||
err := k8sClient.Create(ctx, &updatev1alpha1.NodeImage{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: constants.NodeImageResourceName,
|
||||
},
|
||||
Spec: updatev1alpha1.NodeImageSpec{
|
||||
ImageReference: imageReference,
|
||||
},
|
||||
})
|
||||
if k8sErrors.IsAlreadyExists(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// createScalingGroup creates an initial scaling group resource if it does not exist yet.
|
||||
func createScalingGroup(ctx context.Context, k8sClient client.Writer, groupID, groupName string, autoscaling bool) error {
|
||||
err := k8sClient.Create(ctx, &updatev1alpha1.ScalingGroup{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "ScalingGroup"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: groupName,
|
||||
},
|
||||
Spec: updatev1alpha1.ScalingGroupSpec{
|
||||
NodeImage: constants.NodeImageResourceName,
|
||||
GroupID: groupID,
|
||||
Autoscaling: autoscaling,
|
||||
},
|
||||
})
|
||||
if k8sErrors.IsAlreadyExists(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type scalingGroupGetter interface {
|
||||
// GetScalingGroupImage retrieves the image currently used by a scaling group.
|
||||
GetScalingGroupImage(ctx context.Context, scalingGroupID string) (string, error)
|
||||
// GetScalingGroupName retrieves the name of a scaling group.
|
||||
GetScalingGroupName(ctx context.Context, scalingGroupID string) (string, error)
|
||||
// ListScalingGroups retrieves a list of scaling groups for the cluster.
|
||||
ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error)
|
||||
}
|
||||
|
|
@ -0,0 +1,317 @@
|
|||
package deploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/api/v1alpha1"
|
||||
"github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/constants"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
|
||||
func TestInitialResources(t *testing.T) {
|
||||
testCases := map[string]struct {
|
||||
items []scalingGroupStoreItem
|
||||
imageErr error
|
||||
nameErr error
|
||||
listErr error
|
||||
createErr error
|
||||
wantResources int
|
||||
wantErr bool
|
||||
}{
|
||||
"creating initial resources works": {
|
||||
items: []scalingGroupStoreItem{
|
||||
{groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true},
|
||||
{groupID: "worker", image: "image-1", name: "worker"},
|
||||
},
|
||||
wantResources: 4,
|
||||
},
|
||||
"missing control planes": {
|
||||
items: []scalingGroupStoreItem{
|
||||
{groupID: "worker", image: "image-1", name: "worker"},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"missing workers": {
|
||||
items: []scalingGroupStoreItem{
|
||||
{groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"listing groups fails": {
|
||||
listErr: errors.New("list failed"),
|
||||
wantErr: true,
|
||||
},
|
||||
"creating resources fails": {
|
||||
items: []scalingGroupStoreItem{
|
||||
{groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true},
|
||||
{groupID: "worker", image: "image-1", name: "worker"},
|
||||
},
|
||||
createErr: errors.New("create failed"),
|
||||
wantErr: true,
|
||||
},
|
||||
"getting image fails": {
|
||||
items: []scalingGroupStoreItem{
|
||||
{groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true},
|
||||
{groupID: "worker", image: "image-1", name: "worker"},
|
||||
},
|
||||
imageErr: errors.New("getting image failed"),
|
||||
wantErr: true,
|
||||
},
|
||||
"getting name fails": {
|
||||
items: []scalingGroupStoreItem{
|
||||
{groupID: "control-plane", image: "image-1", name: "control-plane", isControlPlane: true},
|
||||
{groupID: "worker", image: "image-1", name: "worker"},
|
||||
},
|
||||
nameErr: errors.New("getting name failed"),
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range testCases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
require := require.New(t)
|
||||
|
||||
k8sClient := &stubK8sClient{createErr: tc.createErr}
|
||||
scalingGroupGetter := newScalingGroupGetter(tc.items, tc.imageErr, tc.nameErr, tc.listErr)
|
||||
err := InitialResources(context.Background(), k8sClient, scalingGroupGetter, "uid")
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
return
|
||||
}
|
||||
require.NoError(err)
|
||||
assert.Len(k8sClient.createdObjects, tc.wantResources)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateAutoscalingStrategy(t *testing.T) {
|
||||
testCases := map[string]struct {
|
||||
createErr error
|
||||
wantStrategy *updatev1alpha1.AutoscalingStrategy
|
||||
wantErr bool
|
||||
}{
|
||||
"create works": {
|
||||
wantStrategy: &updatev1alpha1.AutoscalingStrategy{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "AutoscalingStrategy"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: constants.AutoscalingStrategyResourceName,
|
||||
},
|
||||
Spec: updatev1alpha1.AutoscalingStrategySpec{
|
||||
Enabled: true,
|
||||
DeploymentName: "constellation-cluster-autoscaler",
|
||||
DeploymentNamespace: "kube-system",
|
||||
},
|
||||
},
|
||||
},
|
||||
"create fails": {
|
||||
createErr: errors.New("create failed"),
|
||||
wantErr: true,
|
||||
},
|
||||
"strategy exists": {
|
||||
createErr: k8sErrors.NewAlreadyExists(schema.GroupResource{}, constants.AutoscalingStrategyResourceName),
|
||||
wantStrategy: &updatev1alpha1.AutoscalingStrategy{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "AutoscalingStrategy"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: constants.AutoscalingStrategyResourceName,
|
||||
},
|
||||
Spec: updatev1alpha1.AutoscalingStrategySpec{
|
||||
Enabled: true,
|
||||
DeploymentName: "constellation-cluster-autoscaler",
|
||||
DeploymentNamespace: "kube-system",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range testCases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
require := require.New(t)
|
||||
|
||||
k8sClient := &stubK8sClient{createErr: tc.createErr}
|
||||
err := createAutoscalingStrategy(context.Background(), k8sClient)
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
return
|
||||
}
|
||||
require.NoError(err)
|
||||
assert.Len(k8sClient.createdObjects, 1)
|
||||
assert.Equal(tc.wantStrategy, k8sClient.createdObjects[0])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateNodeImage(t *testing.T) {
|
||||
testCases := map[string]struct {
|
||||
createErr error
|
||||
wantNodeImage *updatev1alpha1.NodeImage
|
||||
wantErr bool
|
||||
}{
|
||||
"create works": {
|
||||
wantNodeImage: &updatev1alpha1.NodeImage{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: constants.NodeImageResourceName,
|
||||
},
|
||||
Spec: updatev1alpha1.NodeImageSpec{
|
||||
ImageReference: "image-reference",
|
||||
},
|
||||
},
|
||||
},
|
||||
"create fails": {
|
||||
createErr: errors.New("create failed"),
|
||||
wantErr: true,
|
||||
},
|
||||
"image exists": {
|
||||
createErr: k8sErrors.NewAlreadyExists(schema.GroupResource{}, constants.AutoscalingStrategyResourceName),
|
||||
wantNodeImage: &updatev1alpha1.NodeImage{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: constants.NodeImageResourceName,
|
||||
},
|
||||
Spec: updatev1alpha1.NodeImageSpec{
|
||||
ImageReference: "image-reference",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range testCases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
require := require.New(t)
|
||||
|
||||
k8sClient := &stubK8sClient{createErr: tc.createErr}
|
||||
err := createNodeImage(context.Background(), k8sClient, "image-reference")
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
return
|
||||
}
|
||||
require.NoError(err)
|
||||
assert.Len(k8sClient.createdObjects, 1)
|
||||
assert.Equal(tc.wantNodeImage, k8sClient.createdObjects[0])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateScalingGroup(t *testing.T) {
|
||||
testCases := map[string]struct {
|
||||
createErr error
|
||||
wantScalingGroup *updatev1alpha1.ScalingGroup
|
||||
wantErr bool
|
||||
}{
|
||||
"create works": {
|
||||
wantScalingGroup: &updatev1alpha1.ScalingGroup{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "ScalingGroup"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "group-name",
|
||||
},
|
||||
Spec: updatev1alpha1.ScalingGroupSpec{
|
||||
NodeImage: constants.NodeImageResourceName,
|
||||
GroupID: "group-id",
|
||||
Autoscaling: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
"create fails": {
|
||||
createErr: errors.New("create failed"),
|
||||
wantErr: true,
|
||||
},
|
||||
"image exists": {
|
||||
createErr: k8sErrors.NewAlreadyExists(schema.GroupResource{}, constants.AutoscalingStrategyResourceName),
|
||||
wantScalingGroup: &updatev1alpha1.ScalingGroup{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "ScalingGroup"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "group-name",
|
||||
},
|
||||
Spec: updatev1alpha1.ScalingGroupSpec{
|
||||
NodeImage: constants.NodeImageResourceName,
|
||||
GroupID: "group-id",
|
||||
Autoscaling: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range testCases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
require := require.New(t)
|
||||
|
||||
k8sClient := &stubK8sClient{createErr: tc.createErr}
|
||||
err := createScalingGroup(context.Background(), k8sClient, "group-id", "group-name", true)
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
return
|
||||
}
|
||||
require.NoError(err)
|
||||
assert.Len(k8sClient.createdObjects, 1)
|
||||
assert.Equal(tc.wantScalingGroup, k8sClient.createdObjects[0])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type stubK8sClient struct {
|
||||
createdObjects []client.Object
|
||||
createErr error
|
||||
client.Writer
|
||||
}
|
||||
|
||||
func (s *stubK8sClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
|
||||
s.createdObjects = append(s.createdObjects, obj)
|
||||
return s.createErr
|
||||
}
|
||||
|
||||
type stubScalingGroupGetter struct {
|
||||
store map[string]scalingGroupStoreItem
|
||||
imageErr error
|
||||
nameErr error
|
||||
listErr error
|
||||
}
|
||||
|
||||
func newScalingGroupGetter(items []scalingGroupStoreItem, imageErr, nameErr, listErr error) *stubScalingGroupGetter {
|
||||
store := make(map[string]scalingGroupStoreItem)
|
||||
for _, item := range items {
|
||||
store[item.groupID] = item
|
||||
}
|
||||
return &stubScalingGroupGetter{
|
||||
store: store,
|
||||
imageErr: imageErr,
|
||||
nameErr: nameErr,
|
||||
listErr: listErr,
|
||||
}
|
||||
}
|
||||
|
||||
func (g *stubScalingGroupGetter) GetScalingGroupImage(ctx context.Context, scalingGroupID string) (string, error) {
|
||||
return g.store[scalingGroupID].image, g.imageErr
|
||||
}
|
||||
|
||||
func (g *stubScalingGroupGetter) GetScalingGroupName(ctx context.Context, scalingGroupID string) (string, error) {
|
||||
return g.store[scalingGroupID].name, g.nameErr
|
||||
}
|
||||
|
||||
func (g *stubScalingGroupGetter) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) {
|
||||
for _, item := range g.store {
|
||||
if item.isControlPlane {
|
||||
controlPlaneGroupIDs = append(controlPlaneGroupIDs, item.groupID)
|
||||
} else {
|
||||
workerGroupIDs = append(workerGroupIDs, item.groupID)
|
||||
}
|
||||
}
|
||||
return controlPlaneGroupIDs, workerGroupIDs, g.listErr
|
||||
}
|
||||
|
||||
type scalingGroupStoreItem struct {
|
||||
groupID string
|
||||
name string
|
||||
image string
|
||||
isControlPlane bool
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue