diff --git a/operators/constellation-node-operator/controllers/nodeimage_controller.go b/operators/constellation-node-operator/controllers/nodeimage_controller.go index f87edfcde..86f65f304 100644 --- a/operators/constellation-node-operator/controllers/nodeimage_controller.go +++ b/operators/constellation-node-operator/controllers/nodeimage_controller.go @@ -4,6 +4,7 @@ package controllers import ( "context" "reflect" + "strings" "time" nodeutil "github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/node" @@ -28,9 +29,9 @@ import ( const ( // nodeOverprovisionLimit is the maximum number of extra nodes created during the update procedure at any point in time. - nodeOverprovisionLimit = 4 + nodeOverprovisionLimit = 1 // nodeJoinTimeout is the time limit pending nodes have to join the cluster before being terminated. - nodeJoinTimeout = time.Minute * 15 + nodeJoinTimeout = time.Minute * 30 // nodeLeaveTimeout is the time limit pending nodes have to leave the cluster and being terminated. nodeLeaveTimeout = time.Minute donorAnnotation = "constellation.edgeless.systems/donor" @@ -44,7 +45,7 @@ const ( conditionNodeImageOutOfDateMessage = "Some node images are out of date" ) -// NodeImageReconciler reconciles a NodeImage object +// NodeImageReconciler reconciles a NodeImage object. type NodeImageReconciler struct { nodeReplacer etcdRemover @@ -113,7 +114,7 @@ func (r *NodeImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } scalingGroupByID := make(map[string]updatev1alpha1.ScalingGroup, len(scalingGroupList.Items)) for _, scalingGroup := range scalingGroupList.Items { - scalingGroupByID[scalingGroup.Spec.GroupID] = scalingGroup + scalingGroupByID[strings.ToLower(scalingGroup.Spec.GroupID)] = scalingGroup } annotatedNodes, invalidNodes := r.annotateNodes(ctx, nodeList.Items) groups := groupNodes(annotatedNodes, pendingNodeList.Items, desiredNodeImage.Spec.ImageReference) @@ -272,7 +273,7 @@ func (r *NodeImageReconciler) pairDonorsAndHeirs(ctx context.Context, controller // find outdated node in the same group for i := range outdatedNodes { outdatedNode := &outdatedNodes[i] - if outdatedNode.Annotations[scalingGroupAnnotation] != mintNode.pendingNode.Spec.ScalingGroupID || len(outdatedNode.Annotations[heirAnnotation]) != 0 { + if !strings.EqualFold(outdatedNode.Annotations[scalingGroupAnnotation], mintNode.pendingNode.Spec.ScalingGroupID) || len(outdatedNode.Annotations[heirAnnotation]) != 0 { continue } // mark as donor <-> heir pair and delete "pending node" resource @@ -302,6 +303,7 @@ func (r *NodeImageReconciler) pairDonorsAndHeirs(ctx context.Context, controller break } if !foundReplacement { + logr.Info("No replacement found for mint node. Marking as outdated.", "mintNode", mintNode.node.Name, "scalingGroupID", mintNode.pendingNode.Spec.ScalingGroupID) // mint node was not needed as heir. Cleanup obsolete resources. if err := r.Delete(ctx, &mintNode.pendingNode); err != nil { logr.Error(err, "Unable to delete pending node resource", "pendingNode", mintNode.pendingNode.Name) @@ -503,7 +505,7 @@ func (r *NodeImageReconciler) createNewNodes( if len(node.Annotations[heirAnnotation]) != 0 { continue } - outdatedNodesPerScalingGroup[node.Annotations[scalingGroupAnnotation]]++ + outdatedNodesPerScalingGroup[strings.ToLower(node.Annotations[scalingGroupAnnotation])]++ } pendingJoiningNodesPerScalingGroup := make(map[string]int) for _, pendingNode := range pendingNodes { @@ -511,10 +513,11 @@ func (r *NodeImageReconciler) createNewNodes( if pendingNode.Spec.Goal != updatev1alpha1.NodeGoalJoin { continue } - pendingJoiningNodesPerScalingGroup[pendingNode.Spec.ScalingGroupID]++ + pendingJoiningNodesPerScalingGroup[strings.ToLower(pendingNode.Spec.ScalingGroupID)]++ } requiredNodesPerScalingGroup := make(map[string]int, len(outdatedNodesPerScalingGroup)) for scalingGroupID := range outdatedNodesPerScalingGroup { + scalingGroupID := strings.ToLower(scalingGroupID) if pendingJoiningNodesPerScalingGroup[scalingGroupID] < outdatedNodesPerScalingGroup[scalingGroupID] { requiredNodesPerScalingGroup[scalingGroupID] = outdatedNodesPerScalingGroup[scalingGroupID] - pendingJoiningNodesPerScalingGroup[scalingGroupID] } @@ -522,10 +525,10 @@ func (r *NodeImageReconciler) createNewNodes( for scalingGroupID := range requiredNodesPerScalingGroup { scalingGroup, ok := scalingGroupByID[scalingGroupID] if !ok { - logr.Info("Scaling group does not have matching resource", "scalingGroup", scalingGroupID) + logr.Info("Scaling group does not have matching resource", "scalingGroup", scalingGroupID, "scalingGroups", scalingGroupByID) continue } - if scalingGroup.Status.ImageReference != desiredNodeImage.Spec.ImageReference { + if !strings.EqualFold(scalingGroup.Status.ImageReference, desiredNodeImage.Spec.ImageReference) { logr.Info("Scaling group does not use latest image", "scalingGroup", scalingGroupID, "usedImage", scalingGroup.Status.ImageReference, "wantedImage", desiredNodeImage.Spec.ImageReference) continue } @@ -540,7 +543,7 @@ func (r *NodeImageReconciler) createNewNodes( break } logr.Info("Creating new node", "scalingGroup", scalingGroupID) - nodeName, providerID, err := r.CreateNode(ctx, scalingGroupID) + nodeName, providerID, err := r.CreateNode(ctx, scalingGroup.Spec.GroupID) if err != nil { return err } @@ -549,7 +552,7 @@ func (r *NodeImageReconciler) createNewNodes( ObjectMeta: metav1.ObjectMeta{Name: nodeName}, Spec: updatev1alpha1.PendingNodeSpec{ ProviderID: providerID, - ScalingGroupID: scalingGroupID, + ScalingGroupID: scalingGroup.Spec.GroupID, NodeName: nodeName, Goal: updatev1alpha1.NodeGoalJoin, Deadline: &deadline, @@ -757,7 +760,7 @@ func groupNodes(nodes []corev1.Node, pendingNodes []updatev1alpha1.PendingNode, groups.Obsolete = append(groups.Obsolete, node) continue } - if node.Annotations[nodeImageAnnotation] != latestImageReference { + if !strings.EqualFold(node.Annotations[nodeImageAnnotation], latestImageReference) { if heir := node.Annotations[heirAnnotation]; heir != "" { groups.Donors = append(groups.Donors, node) } else { diff --git a/operators/constellation-node-operator/controllers/nodeimage_controller_test.go b/operators/constellation-node-operator/controllers/nodeimage_controller_test.go index d6c1789af..dd6b5fec2 100644 --- a/operators/constellation-node-operator/controllers/nodeimage_controller_test.go +++ b/operators/constellation-node-operator/controllers/nodeimage_controller_test.go @@ -330,6 +330,9 @@ func TestCreateNewNodes(t *testing.T) { "no outdated nodes": { scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{ "scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "scaling-group", + }, Status: updatev1alpha1.ScalingGroupStatus{ ImageReference: "image", }, @@ -350,6 +353,9 @@ func TestCreateNewNodes(t *testing.T) { }, scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{ "scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "scaling-group", + }, Status: updatev1alpha1.ScalingGroupStatus{ ImageReference: "image", }, @@ -371,6 +377,9 @@ func TestCreateNewNodes(t *testing.T) { }, scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{ "scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "scaling-group", + }, Status: updatev1alpha1.ScalingGroupStatus{ ImageReference: "image", }, @@ -392,6 +401,9 @@ func TestCreateNewNodes(t *testing.T) { }, scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{ "scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "scaling-group", + }, Status: updatev1alpha1.ScalingGroupStatus{ ImageReference: "image", }, @@ -411,6 +423,9 @@ func TestCreateNewNodes(t *testing.T) { }, scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{ "scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "scaling-group", + }, Status: updatev1alpha1.ScalingGroupStatus{ ImageReference: "outdated-image", }, @@ -439,6 +454,9 @@ func TestCreateNewNodes(t *testing.T) { }, scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{ "scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "scaling-group", + }, Status: updatev1alpha1.ScalingGroupStatus{ ImageReference: "image", }, @@ -467,6 +485,9 @@ func TestCreateNewNodes(t *testing.T) { }, scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{ "scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "scaling-group", + }, Status: updatev1alpha1.ScalingGroupStatus{ ImageReference: "image", }, @@ -489,6 +510,9 @@ func TestCreateNewNodes(t *testing.T) { }, scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{ "scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "scaling-group", + }, Status: updatev1alpha1.ScalingGroupStatus{ ImageReference: "image", }, @@ -509,11 +533,17 @@ func TestCreateNewNodes(t *testing.T) { }, scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{ "scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "scaling-group", + }, Status: updatev1alpha1.ScalingGroupStatus{ ImageReference: "image", }, }, "other-scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "other-scaling-group", + }, Status: updatev1alpha1.ScalingGroupStatus{ ImageReference: "image", }, diff --git a/operators/constellation-node-operator/controllers/scalinggroup_controller.go b/operators/constellation-node-operator/controllers/scalinggroup_controller.go index 62093d9d0..1fb8be163 100644 --- a/operators/constellation-node-operator/controllers/scalinggroup_controller.go +++ b/operators/constellation-node-operator/controllers/scalinggroup_controller.go @@ -3,6 +3,7 @@ package controllers import ( "context" + "strings" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -75,7 +76,8 @@ func (r *ScalingGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request outdatedCondition := metav1.Condition{ Type: updatev1alpha1.ConditionOutdated, } - if nodeImage == desiredNodeImage.Spec.ImageReference { + imagesMatch := strings.EqualFold(nodeImage, desiredNodeImage.Spec.ImageReference) + if imagesMatch { outdatedCondition.Status = metav1.ConditionFalse outdatedCondition.Reason = conditionScalingGroupUpToDateReason outdatedCondition.Message = conditionScalingGroupUpToDateMessage @@ -90,7 +92,7 @@ func (r *ScalingGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - if nodeImage != desiredNodeImage.Spec.ImageReference { + if !imagesMatch { logr.Info("ScalingGroup NodeImage is out of date") if err := r.scalingGroupUpdater.SetScalingGroupImage(ctx, desiredScalingGroup.Spec.GroupID, desiredNodeImage.Spec.ImageReference); err != nil { logr.Error(err, "Unable to set ScalingGroup NodeImage")