Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-10-01 01:36:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

[node operator] case insensitive equality checks for image and scaling group references

Browse Source
This commit is contained in:
Malte Poll 2022-08-05 12:17:53 +02:00 committed by Malte Poll
parent 80ebfab164
commit fb4bc1545f
3 changed files with 49 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
operators/constellation-node-operator/controllers/nodeimage_controller.go
View File

@ -4,6 +4,7 @@ package controllers
import (
"context"
"reflect"
"strings"
"time"
nodeutil "github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/node"
@ -28,9 +29,9 @@ import (
const (
// nodeOverprovisionLimit is the maximum number of extra nodes created during the update procedure at any point in time.
nodeOverprovisionLimit = 4
nodeOverprovisionLimit = 1
// nodeJoinTimeout is the time limit pending nodes have to join the cluster before being terminated.
nodeJoinTimeout = time.Minute * 15
nodeJoinTimeout = time.Minute * 30
// nodeLeaveTimeout is the time limit pending nodes have to leave the cluster and being terminated.
nodeLeaveTimeout = time.Minute
donorAnnotation = "constellation.edgeless.systems/donor"
@ -44,7 +45,7 @@ const (
conditionNodeImageOutOfDateMessage = "Some node images are out of date"
)
// NodeImageReconciler reconciles a NodeImage object
// NodeImageReconciler reconciles a NodeImage object.
type NodeImageReconciler struct {
nodeReplacer
etcdRemover
@ -113,7 +114,7 @@ func (r *NodeImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
}
scalingGroupByID := make(map[string]updatev1alpha1.ScalingGroup, len(scalingGroupList.Items))
for _, scalingGroup := range scalingGroupList.Items {
scalingGroupByID[scalingGroup.Spec.GroupID] = scalingGroup
scalingGroupByID[strings.ToLower(scalingGroup.Spec.GroupID)] = scalingGroup
}
annotatedNodes, invalidNodes := r.annotateNodes(ctx, nodeList.Items)
groups := groupNodes(annotatedNodes, pendingNodeList.Items, desiredNodeImage.Spec.ImageReference)
@ -272,7 +273,7 @@ func (r *NodeImageReconciler) pairDonorsAndHeirs(ctx context.Context, controller
// find outdated node in the same group
for i := range outdatedNodes {
outdatedNode := &outdatedNodes[i]
if outdatedNode.Annotations[scalingGroupAnnotation] != mintNode.pendingNode.Spec.ScalingGroupID || len(outdatedNode.Annotations[heirAnnotation]) != 0 {
if !strings.EqualFold(outdatedNode.Annotations[scalingGroupAnnotation], mintNode.pendingNode.Spec.ScalingGroupID) || len(outdatedNode.Annotations[heirAnnotation]) != 0 {
continue
}
// mark as donor <-> heir pair and delete "pending node" resource
@ -302,6 +303,7 @@ func (r *NodeImageReconciler) pairDonorsAndHeirs(ctx context.Context, controller
break
}
if !foundReplacement {
logr.Info("No replacement found for mint node. Marking as outdated.", "mintNode", mintNode.node.Name, "scalingGroupID", mintNode.pendingNode.Spec.ScalingGroupID)
// mint node was not needed as heir. Cleanup obsolete resources.
if err := r.Delete(ctx, &mintNode.pendingNode); err != nil {
logr.Error(err, "Unable to delete pending node resource", "pendingNode", mintNode.pendingNode.Name)
@ -503,7 +505,7 @@ func (r *NodeImageReconciler) createNewNodes(
if len(node.Annotations[heirAnnotation]) != 0 {
continue
}
outdatedNodesPerScalingGroup[node.Annotations[scalingGroupAnnotation]]++
outdatedNodesPerScalingGroup[strings.ToLower(node.Annotations[scalingGroupAnnotation])]++
}
pendingJoiningNodesPerScalingGroup := make(map[string]int)
for _, pendingNode := range pendingNodes {
@ -511,10 +513,11 @@ func (r *NodeImageReconciler) createNewNodes(
if pendingNode.Spec.Goal != updatev1alpha1.NodeGoalJoin {
continue
}
pendingJoiningNodesPerScalingGroup[pendingNode.Spec.ScalingGroupID]++
pendingJoiningNodesPerScalingGroup[strings.ToLower(pendingNode.Spec.ScalingGroupID)]++
}
requiredNodesPerScalingGroup := make(map[string]int, len(outdatedNodesPerScalingGroup))
for scalingGroupID := range outdatedNodesPerScalingGroup {
scalingGroupID := strings.ToLower(scalingGroupID)
if pendingJoiningNodesPerScalingGroup[scalingGroupID] < outdatedNodesPerScalingGroup[scalingGroupID] {
requiredNodesPerScalingGroup[scalingGroupID] = outdatedNodesPerScalingGroup[scalingGroupID] - pendingJoiningNodesPerScalingGroup[scalingGroupID]
}
@ -522,10 +525,10 @@ func (r *NodeImageReconciler) createNewNodes(
for scalingGroupID := range requiredNodesPerScalingGroup {
scalingGroup, ok := scalingGroupByID[scalingGroupID]
if !ok {
logr.Info("Scaling group does not have matching resource", "scalingGroup", scalingGroupID)
logr.Info("Scaling group does not have matching resource", "scalingGroup", scalingGroupID, "scalingGroups", scalingGroupByID)
continue
}
if scalingGroup.Status.ImageReference != desiredNodeImage.Spec.ImageReference {
if !strings.EqualFold(scalingGroup.Status.ImageReference, desiredNodeImage.Spec.ImageReference) {
logr.Info("Scaling group does not use latest image", "scalingGroup", scalingGroupID, "usedImage", scalingGroup.Status.ImageReference, "wantedImage", desiredNodeImage.Spec.ImageReference)
continue
}
@ -540,7 +543,7 @@ func (r *NodeImageReconciler) createNewNodes(
break
}
logr.Info("Creating new node", "scalingGroup", scalingGroupID)
nodeName, providerID, err := r.CreateNode(ctx, scalingGroupID)
nodeName, providerID, err := r.CreateNode(ctx, scalingGroup.Spec.GroupID)
if err != nil {
return err
}
@ -549,7 +552,7 @@ func (r *NodeImageReconciler) createNewNodes(
ObjectMeta: metav1.ObjectMeta{Name: nodeName},
Spec: updatev1alpha1.PendingNodeSpec{
ProviderID: providerID,
ScalingGroupID: scalingGroupID,
ScalingGroupID: scalingGroup.Spec.GroupID,
NodeName: nodeName,
Goal: updatev1alpha1.NodeGoalJoin,
Deadline: &deadline,
@ -757,7 +760,7 @@ func groupNodes(nodes []corev1.Node, pendingNodes []updatev1alpha1.PendingNode,
groups.Obsolete = append(groups.Obsolete, node)
continue
}
if node.Annotations[nodeImageAnnotation] != latestImageReference {
if !strings.EqualFold(node.Annotations[nodeImageAnnotation], latestImageReference) {
if heir := node.Annotations[heirAnnotation]; heir != "" {
groups.Donors = append(groups.Donors, node)
} else {

30
operators/constellation-node-operator/controllers/nodeimage_controller_test.go
View File

@ -330,6 +330,9 @@ func TestCreateNewNodes(t *testing.T) {
"no outdated nodes": {
scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{
"scaling-group": {
Spec: updatev1alpha1.ScalingGroupSpec{
GroupID: "scaling-group",
},
Status: updatev1alpha1.ScalingGroupStatus{
ImageReference: "image",
},
@ -350,6 +353,9 @@ func TestCreateNewNodes(t *testing.T) {
},
scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{
"scaling-group": {
Spec: updatev1alpha1.ScalingGroupSpec{
GroupID: "scaling-group",
},
Status: updatev1alpha1.ScalingGroupStatus{
ImageReference: "image",
},
@ -371,6 +377,9 @@ func TestCreateNewNodes(t *testing.T) {
},
scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{
"scaling-group": {
Spec: updatev1alpha1.ScalingGroupSpec{
GroupID: "scaling-group",
},
Status: updatev1alpha1.ScalingGroupStatus{
ImageReference: "image",
},
@ -392,6 +401,9 @@ func TestCreateNewNodes(t *testing.T) {
},
scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{
"scaling-group": {
Spec: updatev1alpha1.ScalingGroupSpec{
GroupID: "scaling-group",
},
Status: updatev1alpha1.ScalingGroupStatus{
ImageReference: "image",
},
@ -411,6 +423,9 @@ func TestCreateNewNodes(t *testing.T) {
},
scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{
"scaling-group": {
Spec: updatev1alpha1.ScalingGroupSpec{
GroupID: "scaling-group",
},
Status: updatev1alpha1.ScalingGroupStatus{
ImageReference: "outdated-image",
},
@ -439,6 +454,9 @@ func TestCreateNewNodes(t *testing.T) {
},
scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{
"scaling-group": {
Spec: updatev1alpha1.ScalingGroupSpec{
GroupID: "scaling-group",
},
Status: updatev1alpha1.ScalingGroupStatus{
ImageReference: "image",
},
@ -467,6 +485,9 @@ func TestCreateNewNodes(t *testing.T) {
},
scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{
"scaling-group": {
Spec: updatev1alpha1.ScalingGroupSpec{
GroupID: "scaling-group",
},
Status: updatev1alpha1.ScalingGroupStatus{
ImageReference: "image",
},
@ -489,6 +510,9 @@ func TestCreateNewNodes(t *testing.T) {
},
scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{
"scaling-group": {
Spec: updatev1alpha1.ScalingGroupSpec{
GroupID: "scaling-group",
},
Status: updatev1alpha1.ScalingGroupStatus{
ImageReference: "image",
},
@ -509,11 +533,17 @@ func TestCreateNewNodes(t *testing.T) {
},
scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{
"scaling-group": {
Spec: updatev1alpha1.ScalingGroupSpec{
GroupID: "scaling-group",
},
Status: updatev1alpha1.ScalingGroupStatus{
ImageReference: "image",
},
},
"other-scaling-group": {
Spec: updatev1alpha1.ScalingGroupSpec{
GroupID: "other-scaling-group",
},
Status: updatev1alpha1.ScalingGroupStatus{
ImageReference: "image",
},

6
operators/constellation-node-operator/controllers/scalinggroup_controller.go
View File

@ -3,6 +3,7 @@ package controllers
import (
"context"
"strings"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -75,7 +76,8 @@ func (r *ScalingGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request
outdatedCondition := metav1.Condition{
Type: updatev1alpha1.ConditionOutdated,
}
if nodeImage == desiredNodeImage.Spec.ImageReference {
imagesMatch := strings.EqualFold(nodeImage, desiredNodeImage.Spec.ImageReference)
if imagesMatch {
outdatedCondition.Status = metav1.ConditionFalse
outdatedCondition.Reason = conditionScalingGroupUpToDateReason
outdatedCondition.Message = conditionScalingGroupUpToDateMessage
@ -90,7 +92,7 @@ func (r *ScalingGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, err
}
if nodeImage != desiredNodeImage.Spec.ImageReference {
if !imagesMatch {
logr.Info("ScalingGroup NodeImage is out of date")
if err := r.scalingGroupUpdater.SetScalingGroupImage(ctx, desiredScalingGroup.Spec.GroupID, desiredNodeImage.Spec.ImageReference); err != nil {
logr.Error(err, "Unable to set ScalingGroup NodeImage")