operators: ignore node deletion errors on absence (#3113)

* operators: ignore node deletion errors on absence

operators/constellation-node-operator/internal/cloud/aws/client/nodeimage.go

@@ -9,6 +9,7 @@ package client
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/autoscaling"
@@ -207,7 +208,7 @@ func (c *Client) DeleteNode(ctx context.Context, providerID string) error {
 			ShouldDecrementDesiredCapacity: toPtr(true),
 		},
 	)
-	if err != nil {
+	if err != nil && !isInstanceNotFoundError(err) {
 		return fmt.Errorf("failed to terminate instance: %w", err)
 	}
 
@@ -217,3 +218,10 @@ func (c *Client) DeleteNode(ctx context.Context, providerID string) error {
 func toPtr[T any](v T) *T {
 	return &v
 }
+
+func isInstanceNotFoundError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), "Instance Id not found")
+}

operators/constellation-node-operator/internal/cloud/aws/client/nodeimage_test.go

@@ -8,6 +8,7 @@ package client
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"github.com/aws/aws-sdk-go-v2/service/autoscaling"
@@ -382,6 +383,10 @@ func TestDeleteNode(t *testing.T) {
 			terminateInstanceErr: assert.AnError,
 			wantErr:              true,
 		},
+		"deleting node succeeds when the instance does not exist": {
+			providerID:           "aws:///us-east-2a/i-00000000000000000",
+			terminateInstanceErr: fmt.Errorf("Instance Id not found - No managed instance found for instance ID: i-00000000000000000"),
+		},
 	}
 	for name, tc := range testCases {
 		t.Run(name, func(t *testing.T) {

operators/constellation-node-operator/internal/cloud/gcp/client/nodeimage.go

@@ -139,7 +139,8 @@ func (c *Client) DeleteNode(ctx context.Context, providerID string) error {
 		Project:              instanceGroupProject,
 		Zone:                 instanceGroupZone,
 		InstanceGroupManagersDeleteInstancesRequestResource: &computepb.InstanceGroupManagersDeleteInstancesRequest{
-			Instances: []string{instanceID},
+			Instances:                      []string{instanceID},
+			SkipInstancesOnValidationError: toPtr(true),
 		},
 	})
 	if err != nil {
@@ -147,3 +148,7 @@ func (c *Client) DeleteNode(ctx context.Context, providerID string) error {
 	}
 	return op.Wait(ctx)
 }
+
+func toPtr[T any](v T) *T {
+	return &v
+}