operator: make test more resilient (#959)

operators/constellation-node-operator/controllers/autoscalingstrategy_controller_env_test.go

@@ -148,9 +148,13 @@ var _ = Describe("AutoscalingStrategy controller", func() {
 			}, timeout, interval).Should(Equal(int32(1)))
 
 			By("disabling the autoscaler in the strategy")
-			Expect(k8sClient.Get(ctx, strategyLookupKey, strategy)).Should(Succeed())
+			Eventually(func() error {
+				if err := k8sClient.Get(ctx, strategyLookupKey, strategy); err != nil {
+					return err
+				}
 				strategy.Spec.Enabled = false
-			Expect(k8sClient.Update(ctx, strategy)).Should(Succeed())
+				return k8sClient.Update(ctx, strategy)
+			}, timeout, interval).Should(Succeed())
 
 			By("checking the autoscaling deployment eventually has zero replicas")
 			Eventually(checkDeploymentReplicas, timeout, interval).Should(Equal(int32(0)))

operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go

@@ -169,8 +169,13 @@ var _ = Describe("JoiningNode controller", func() {
 		Expect(createdJoiningNode.Spec.ComponentsReference).Should(Equal(ComponentsReference3))
 
 		By("setting the deadline to the past")
+		Eventually(func() error {
+			if err := k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode); err != nil {
+				return err
+			}
 			createdJoiningNode.Spec.Deadline = &metav1.Time{Time: fakes.clock.Now().Add(-time.Second)}
-		Expect(k8sClient.Update(ctx, createdJoiningNode)).Should(Succeed())
+			return k8sClient.Update(ctx, createdJoiningNode)
+		}, timeout, interval).Should(Succeed())
 
 		By("deleting the joining node resource")
 		Eventually(func() error {

operators/constellation-node-operator/controllers/nodeversion_controller_env_test.go

@@ -276,9 +276,19 @@ var _ = Describe("NodeVersion controller", func() {
 			}, timeout, interval).Should(HaveKeyWithValue(heirAnnotation, secondNodeName))
 			Expect(k8sClient.Get(ctx, secondNodeLookupKey, secondNode)).Should(Succeed())
 			Expect(secondNode.Annotations).Should(HaveKeyWithValue(donorAnnotation, firstNodeName))
-			Expect(k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion)).Should(Succeed())
+
-			Expect(nodeVersion.Status.Donors).Should(HaveLen(1))
+			Eventually(func() error {
-			Expect(nodeVersion.Status.Heirs).Should(HaveLen(1))
+				if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil {
+					return err
+				}
+				if len(nodeVersion.Status.Donors) != 1 {
+					return fmt.Errorf("node version %s has %d donors, expected 1", nodeVersion.Name, len(nodeVersion.Status.Donors))
+				}
+				if len(nodeVersion.Status.Heirs) != 1 {
+					return fmt.Errorf("node version %s has %d heirs, expected 1", nodeVersion.Name, len(nodeVersion.Status.Heirs))
+				}
+				return nil
+			}, timeout, interval).Should(Succeed())
 			Expect(k8sClient.Get(ctx, joiningPendingNodeLookupKey, pendingNode)).Should(Not(Succeed()))
 
 			By("checking that node labels are copied to the heir")

operators/constellation-node-operator/controllers/pendingnode_controller_env_test.go

@@ -77,10 +77,13 @@ var _ = Describe("PendingNode controller", func() {
 			}, timeout, interval).Should(Equal(updatev1alpha1.NodeStateCreating))
 
 			By("updating the deadline to be in the past")
-			deadline := fakes.clock.Now().Add(-time.Second)
+			Eventually(func() error {
-			Expect(k8sClient.Get(ctx, pendingNodeLookupKey, pendingNode)).Should(Succeed())
+				if err := k8sClient.Get(ctx, pendingNodeLookupKey, pendingNode); err != nil {
-			pendingNode.Spec.Deadline = &metav1.Time{Time: deadline}
+					return err
-			Expect(k8sClient.Update(ctx, pendingNode)).Should(Succeed())
+				}
+				pendingNode.Spec.Deadline = &metav1.Time{Time: fakes.clock.Now().Add(-time.Second)}
+				return k8sClient.Update(ctx, pendingNode)
+			}, timeout, interval).Should(Succeed())
 
 			By("checking the pending node updates its goal")
 			Eventually(func() updatev1alpha1.PendingNodeGoal {

operators/constellation-node-operator/controllers/scalinggroup_controller_env_test.go

@@ -90,9 +90,13 @@ var _ = Describe("ScalingGroup controller", func() {
 			}, timeout, interval).Should(Equal("image-1"))
 
 			By("updating the node image")
-			Expect(k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion)).Should(Succeed())
+			Eventually(func() error {
+				if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil {
+					return err
+				}
 				nodeVersion.Spec.ImageReference = "image-2"
-			Expect(k8sClient.Update(ctx, nodeVersion)).Should(Succeed())
+				return k8sClient.Update(ctx, nodeVersion)
+			}, timeout, interval).Should(Succeed())
 
 			By("checking the scaling group eventually uses the latest image")
 			Eventually(func() string {