mirror of
https://github.com/edgelesssys/constellation.git
synced 2025-05-02 06:16:08 -04:00
upgrade: support Kubernetes components (#839)
* upgrade: add Kubernetes components to NodeVersion * update rfc
This commit is contained in:
3u13r
GitHub
parent
4b43311fbd
commit
f14af0c3eb
56 changed files with 897 additions and 738 deletions
|
|
@ -12,9 +12,11 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
|
||||
"github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/internal/constants"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
|
@ -22,7 +24,7 @@ import (
|
|||
)
|
||||
|
||||
// InitialResources creates the initial resources for the node operator.
|
||||
func InitialResources(ctx context.Context, k8sClient client.Writer, imageInfo imageInfoGetter, scalingGroupGetter scalingGroupGetter, uid string) error {
|
||||
func InitialResources(ctx context.Context, k8sClient client.Client, imageInfo imageInfoGetter, scalingGroupGetter scalingGroupGetter, uid string) error {
|
||||
logr := log.FromContext(ctx)
|
||||
controlPlaneGroupIDs, workerGroupIDs, err := scalingGroupGetter.ListScalingGroups(ctx, uid)
|
||||
if err != nil {
|
||||
|
|
@ -50,8 +52,8 @@ func InitialResources(ctx context.Context, k8sClient client.Writer, imageInfo im
|
|||
imageVersion = ""
|
||||
}
|
||||
|
||||
if err := createNodeImage(ctx, k8sClient, imageReference, imageVersion); err != nil {
|
||||
return fmt.Errorf("creating initial node image %q: %w", imageReference, err)
|
||||
if err := createNodeVersion(ctx, k8sClient, imageReference, imageVersion); err != nil {
|
||||
return fmt.Errorf("creating initial node version %q: %w", imageReference, err)
|
||||
}
|
||||
for _, groupID := range controlPlaneGroupIDs {
|
||||
groupName, err := scalingGroupGetter.GetScalingGroupName(groupID)
|
||||
|
|
@ -110,22 +112,61 @@ func createAutoscalingStrategy(ctx context.Context, k8sClient client.Writer, pro
|
|||
return err
|
||||
}
|
||||
|
||||
// createNodeImage creates the initial nodeimage resource if it does not exist yet.
|
||||
func createNodeImage(ctx context.Context, k8sClient client.Writer, imageReference, imageVersion string) error {
|
||||
err := k8sClient.Create(ctx, &updatev1alpha1.NodeImage{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"},
|
||||
// createNodeVersion creates the initial nodeversion resource if it does not exist yet.
|
||||
func createNodeVersion(ctx context.Context, k8sClient client.Client, imageReference, imageVersion string) error {
|
||||
k8sComponentsRef, err := findLatestK8sComponentsConfigMap(ctx, k8sClient)
|
||||
if err != nil {
|
||||
return fmt.Errorf("finding latest k8s-components configmap: %w", err)
|
||||
}
|
||||
err = k8sClient.Create(ctx, &updatev1alpha1.NodeVersion{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeVersion"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: constants.NodeImageResourceName,
|
||||
Name: constants.NodeVersionResourceName,
|
||||
},
|
||||
Spec: updatev1alpha1.NodeImageSpec{
|
||||
ImageReference: imageReference,
|
||||
ImageVersion: imageVersion,
|
||||
Spec: updatev1alpha1.NodeVersionSpec{
|
||||
ImageReference: imageReference,
|
||||
ImageVersion: imageVersion,
|
||||
KubernetesComponentsReference: k8sComponentsRef,
|
||||
},
|
||||
})
|
||||
if k8sErrors.IsAlreadyExists(err) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return err
|
||||
return nil
|
||||
}
|
||||
|
||||
// findLatestK8sComponentsConfigMap finds most recently created k8s-components configmap in the kube-system namespace.
|
||||
// It returns an error if there is no or multiple configmaps matching the prefix "k8s-components".
|
||||
func findLatestK8sComponentsConfigMap(ctx context.Context, k8sClient client.Client) (string, error) {
|
||||
var configMaps corev1.ConfigMapList
|
||||
err := k8sClient.List(ctx, &configMaps, client.InNamespace("kube-system"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("listing configmaps: %w", err)
|
||||
}
|
||||
|
||||
// collect all k8s-components configmaps
|
||||
componentConfigMaps := make(map[string]time.Time)
|
||||
for _, configMap := range configMaps.Items {
|
||||
if strings.HasPrefix(configMap.Name, "k8s-components") {
|
||||
componentConfigMaps[configMap.Name] = configMap.CreationTimestamp.Time
|
||||
}
|
||||
}
|
||||
if len(componentConfigMaps) == 0 {
|
||||
return "", fmt.Errorf("no configmaps found")
|
||||
}
|
||||
|
||||
// find latest configmap
|
||||
var latestConfigMap string
|
||||
var latestTime time.Time
|
||||
for configMap, creationTime := range componentConfigMaps {
|
||||
if creationTime.After(latestTime) {
|
||||
latestConfigMap = configMap
|
||||
latestTime = creationTime
|
||||
}
|
||||
}
|
||||
return latestConfigMap, nil
|
||||
}
|
||||
|
||||
// createScalingGroup creates an initial scaling group resource if it does not exist yet.
|
||||
|
|
@ -136,7 +177,7 @@ func createScalingGroup(ctx context.Context, config newScalingGroupConfig) error
|
|||
Name: strings.ToLower(config.groupName),
|
||||
},
|
||||
Spec: updatev1alpha1.ScalingGroupSpec{
|
||||
NodeImage: constants.NodeImageResourceName,
|
||||
NodeVersion: constants.NodeVersionResourceName,
|
||||
GroupID: config.groupID,
|
||||
AutoscalerGroupName: config.autoscalingGroupName,
|
||||
Min: 1,
|
||||
|
|
|
|||
|
|
@ -10,18 +10,22 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
|
||||
"github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/internal/constants"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
|
||||
func TestInitialResources(t *testing.T) {
|
||||
k8sComponentsReference := "k8s-components-sha256-ABC"
|
||||
testCases := map[string]struct {
|
||||
items []scalingGroupStoreItem
|
||||
imageErr error
|
||||
|
|
@ -85,7 +89,16 @@ func TestInitialResources(t *testing.T) {
|
|||
assert := assert.New(t)
|
||||
require := require.New(t)
|
||||
|
||||
k8sClient := &stubK8sClient{createErr: tc.createErr}
|
||||
k8sClient := &fakeK8sClient{
|
||||
createErr: tc.createErr,
|
||||
listConfigMaps: []corev1.ConfigMap{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: k8sComponentsReference,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
scalingGroupGetter := newScalingGroupGetter(tc.items, tc.imageErr, tc.nameErr, tc.listErr)
|
||||
err := InitialResources(context.Background(), k8sClient, &stubImageInfo{}, scalingGroupGetter, "uid")
|
||||
if tc.wantErr {
|
||||
|
|
@ -156,7 +169,7 @@ func TestCreateAutoscalingStrategy(t *testing.T) {
|
|||
assert := assert.New(t)
|
||||
require := require.New(t)
|
||||
|
||||
k8sClient := &stubK8sClient{createErr: tc.createErr}
|
||||
k8sClient := &fakeK8sClient{createErr: tc.createErr}
|
||||
err := createAutoscalingStrategy(context.Background(), k8sClient, "stub")
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
|
|
@ -169,21 +182,24 @@ func TestCreateAutoscalingStrategy(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestCreateNodeImage(t *testing.T) {
|
||||
func TestCreateNodeVersion(t *testing.T) {
|
||||
k8sComponentsReference := "k8s-components-sha256-reference"
|
||||
testCases := map[string]struct {
|
||||
createErr error
|
||||
wantNodeImage *updatev1alpha1.NodeImage
|
||||
wantErr bool
|
||||
createErr error
|
||||
existingNodeVersion *updatev1alpha1.NodeVersion
|
||||
wantNodeVersion *updatev1alpha1.NodeVersion
|
||||
wantErr bool
|
||||
}{
|
||||
"create works": {
|
||||
wantNodeImage: &updatev1alpha1.NodeImage{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"},
|
||||
wantNodeVersion: &updatev1alpha1.NodeVersion{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeVersion"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: constants.NodeImageResourceName,
|
||||
Name: constants.NodeVersionResourceName,
|
||||
},
|
||||
Spec: updatev1alpha1.NodeImageSpec{
|
||||
ImageReference: "image-reference",
|
||||
ImageVersion: "image-version",
|
||||
Spec: updatev1alpha1.NodeVersionSpec{
|
||||
ImageReference: "image-reference",
|
||||
ImageVersion: "image-version",
|
||||
KubernetesComponentsReference: k8sComponentsReference,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -191,16 +207,28 @@ func TestCreateNodeImage(t *testing.T) {
|
|||
createErr: errors.New("create failed"),
|
||||
wantErr: true,
|
||||
},
|
||||
"image exists": {
|
||||
createErr: k8sErrors.NewAlreadyExists(schema.GroupResource{}, constants.AutoscalingStrategyResourceName),
|
||||
wantNodeImage: &updatev1alpha1.NodeImage{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"},
|
||||
"version exists": {
|
||||
createErr: k8sErrors.NewAlreadyExists(schema.GroupResource{}, constants.NodeVersionResourceName),
|
||||
existingNodeVersion: &updatev1alpha1.NodeVersion{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeVersion"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: constants.NodeImageResourceName,
|
||||
Name: constants.NodeVersionResourceName,
|
||||
},
|
||||
Spec: updatev1alpha1.NodeImageSpec{
|
||||
ImageReference: "image-reference",
|
||||
ImageVersion: "image-version",
|
||||
Spec: updatev1alpha1.NodeVersionSpec{
|
||||
ImageReference: "image-reference2",
|
||||
ImageVersion: "image-version2",
|
||||
KubernetesComponentsReference: "components-reference2",
|
||||
},
|
||||
},
|
||||
wantNodeVersion: &updatev1alpha1.NodeVersion{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeVersion"},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: constants.NodeVersionResourceName,
|
||||
},
|
||||
Spec: updatev1alpha1.NodeVersionSpec{
|
||||
ImageReference: "image-reference2",
|
||||
ImageVersion: "image-version2",
|
||||
KubernetesComponentsReference: "components-reference2",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -211,15 +239,28 @@ func TestCreateNodeImage(t *testing.T) {
|
|||
assert := assert.New(t)
|
||||
require := require.New(t)
|
||||
|
||||
k8sClient := &stubK8sClient{createErr: tc.createErr}
|
||||
err := createNodeImage(context.Background(), k8sClient, "image-reference", "image-version")
|
||||
k8sClient := &fakeK8sClient{
|
||||
createErr: tc.createErr,
|
||||
listConfigMaps: []corev1.ConfigMap{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: k8sComponentsReference,
|
||||
CreationTimestamp: metav1.Time{Time: time.Unix(1, 0)},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if tc.existingNodeVersion != nil {
|
||||
k8sClient.createdObjects = append(k8sClient.createdObjects, tc.existingNodeVersion)
|
||||
}
|
||||
err := createNodeVersion(context.Background(), k8sClient, "image-reference", "image-version")
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
return
|
||||
}
|
||||
require.NoError(err)
|
||||
assert.Len(k8sClient.createdObjects, 1)
|
||||
assert.Equal(tc.wantNodeImage, k8sClient.createdObjects[0])
|
||||
assert.Equal(tc.wantNodeVersion, k8sClient.createdObjects[0])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -237,7 +278,7 @@ func TestCreateScalingGroup(t *testing.T) {
|
|||
Name: "group-name",
|
||||
},
|
||||
Spec: updatev1alpha1.ScalingGroupSpec{
|
||||
NodeImage: constants.NodeImageResourceName,
|
||||
NodeVersion: constants.NodeVersionResourceName,
|
||||
GroupID: "group-id",
|
||||
AutoscalerGroupName: "group-Name",
|
||||
Min: 1,
|
||||
|
|
@ -258,7 +299,7 @@ func TestCreateScalingGroup(t *testing.T) {
|
|||
Name: "group-name",
|
||||
},
|
||||
Spec: updatev1alpha1.ScalingGroupSpec{
|
||||
NodeImage: constants.NodeImageResourceName,
|
||||
NodeVersion: constants.NodeVersionResourceName,
|
||||
GroupID: "group-id",
|
||||
AutoscalerGroupName: "group-Name",
|
||||
Min: 1,
|
||||
|
|
@ -274,7 +315,7 @@ func TestCreateScalingGroup(t *testing.T) {
|
|||
assert := assert.New(t)
|
||||
require := require.New(t)
|
||||
|
||||
k8sClient := &stubK8sClient{createErr: tc.createErr}
|
||||
k8sClient := &fakeK8sClient{createErr: tc.createErr}
|
||||
newScalingGroupConfig := newScalingGroupConfig{k8sClient, "group-id", "group-Name", "group-Name", updatev1alpha1.WorkerRole}
|
||||
err := createScalingGroup(context.Background(), newScalingGroupConfig)
|
||||
if tc.wantErr {
|
||||
|
|
@ -288,17 +329,65 @@ func TestCreateScalingGroup(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
type stubK8sClient struct {
|
||||
type fakeK8sClient struct {
|
||||
createdObjects []client.Object
|
||||
createErr error
|
||||
client.Writer
|
||||
listConfigMaps []corev1.ConfigMap
|
||||
listErr error
|
||||
getErr error
|
||||
updateErr error
|
||||
client.Client
|
||||
}
|
||||
|
||||
func (s *stubK8sClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
|
||||
func (s *fakeK8sClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
|
||||
for _, o := range s.createdObjects {
|
||||
if obj.GetName() == o.GetName() {
|
||||
return k8sErrors.NewAlreadyExists(schema.GroupResource{}, obj.GetName())
|
||||
}
|
||||
}
|
||||
|
||||
s.createdObjects = append(s.createdObjects, obj)
|
||||
return s.createErr
|
||||
}
|
||||
|
||||
func (s *fakeK8sClient) Get(ctx context.Context, key types.NamespacedName, obj client.Object, opts ...client.GetOption) error {
|
||||
if ObjNodeVersion, ok := obj.(*updatev1alpha1.NodeVersion); ok {
|
||||
for _, o := range s.createdObjects {
|
||||
if createdNodeVersion, ok := o.(*updatev1alpha1.NodeVersion); ok && createdNodeVersion != nil {
|
||||
if createdNodeVersion.Name == key.Name {
|
||||
ObjNodeVersion.ObjectMeta = createdNodeVersion.ObjectMeta
|
||||
ObjNodeVersion.TypeMeta = createdNodeVersion.TypeMeta
|
||||
ObjNodeVersion.Spec = createdNodeVersion.Spec
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return s.getErr
|
||||
}
|
||||
|
||||
func (s *fakeK8sClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
|
||||
if updatedObjectNodeVersion, ok := obj.(*updatev1alpha1.NodeVersion); ok {
|
||||
for i, o := range s.createdObjects {
|
||||
if createdObjectNodeVersion, ok := o.(*updatev1alpha1.NodeVersion); ok && createdObjectNodeVersion != nil {
|
||||
if createdObjectNodeVersion.Name == updatedObjectNodeVersion.Name {
|
||||
s.createdObjects[i] = obj
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return s.updateErr
|
||||
}
|
||||
|
||||
func (s *fakeK8sClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
|
||||
if configMapList, ok := list.(*corev1.ConfigMapList); ok {
|
||||
configMapList.Items = append(configMapList.Items, s.listConfigMaps...)
|
||||
}
|
||||
return s.listErr
|
||||
}
|
||||
|
||||
type stubImageInfo struct {
|
||||
imageVersion string
|
||||
err error
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue