Check for 404 errors in GCP termination

cli/internal/gcp/client/client.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"math/big"
+	"net/http"
 
 	compute "cloud.google.com/go/compute/apiv1"
 	admin "cloud.google.com/go/iam/admin/apiv1"
@@ -15,6 +17,7 @@ import (
 	"github.com/edgelesssys/constellation/internal/state"
 	"go.uber.org/multierr"
 	"golang.org/x/oauth2/google"
+	"google.golang.org/api/googleapi"
 )
 
 // Client is a client for the Google Compute Engine.
@@ -318,3 +321,11 @@ func closeAll(closers []closer) error {
 	}
 	return err
 }
+
+func isNotFoundError(err error) bool {
+	var gAPIErr *googleapi.Error
+	if !errors.As(err, &gAPIErr) {
+		return false
+	}
+	return gAPIErr.Code == http.StatusNotFound
+}

cli/internal/gcp/client/instances.go

@@ -131,20 +131,24 @@ func (c *Client) TerminateInstances(ctx context.Context) error {
 	ops := []Operation{}
 	if c.workerInstanceGroup != "" {
 		op, err := c.deleteInstanceGroupManager(ctx, c.workerInstanceGroup)
-		if err != nil {
+		if err != nil && !isNotFoundError(err) {
 			return fmt.Errorf("deleting instanceGroupManager '%s': %w", c.workerInstanceGroup, err)
 		}
+		if err == nil {
 			ops = append(ops, op)
+		}
 		c.workerInstanceGroup = ""
 		c.workers = make(cloudtypes.Instances)
 	}
 
 	if c.controlPlaneInstanceGroup != "" {
 		op, err := c.deleteInstanceGroupManager(ctx, c.controlPlaneInstanceGroup)
-		if err != nil {
+		if err != nil && !isNotFoundError(err) {
 			return fmt.Errorf("deleting instanceGroupManager '%s': %w", c.controlPlaneInstanceGroup, err)
 		}
+		if err == nil {
 			ops = append(ops, op)
+		}
 		c.controlPlaneInstanceGroup = ""
 		c.controlPlanes = make(cloudtypes.Instances)
 	}
@@ -155,18 +159,22 @@ func (c *Client) TerminateInstances(ctx context.Context) error {
 
 	if c.workerTemplate != "" {
 		op, err := c.deleteInstanceTemplate(ctx, c.workerTemplate)
-		if err != nil {
+		if err != nil && !isNotFoundError(err) {
 			return fmt.Errorf("deleting instanceTemplate: %w", err)
 		}
+		if err == nil {
 			ops = append(ops, op)
+		}
 		c.workerTemplate = ""
 	}
 	if c.controlPlaneTemplate != "" {
 		op, err := c.deleteInstanceTemplate(ctx, c.controlPlaneTemplate)
-		if err != nil {
+		if err != nil && !isNotFoundError(err) {
 			return fmt.Errorf("deleting instanceTemplate: %w", err)
 		}
+		if err == nil {
 			ops = append(ops, op)
+		}
 		c.controlPlaneTemplate = ""
 	}
 	return c.waitForOperations(ctx, ops)

cli/internal/gcp/client/instances_test.go

@@ -3,10 +3,12 @@ package client
 import (
 	"context"
 	"errors"
+	"net/http"
 	"testing"
 
 	"github.com/edgelesssys/constellation/internal/cloud/cloudtypes"
 	"github.com/stretchr/testify/assert"
+	"google.golang.org/api/googleapi"
 	computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
 	"google.golang.org/protobuf/proto"
 )
@@ -180,6 +182,8 @@ func TestCreateInstances(t *testing.T) {
 
 func TestTerminateInstances(t *testing.T) {
 	someErr := errors.New("failed")
+	notFoundErr := &googleapi.Error{Code: http.StatusNotFound}
+
 	testCases := map[string]struct {
 		operationZoneAPI         operationZoneAPI
 		operationGlobalAPI       operationGlobalAPI
@@ -202,6 +206,18 @@ func TestTerminateInstances(t *testing.T) {
 			instanceGroupManagersAPI:   stubInstanceGroupManagersAPI{},
 			missingWorkerInstanceGroup: true,
 		},
+		"instances not found": {
+			operationZoneAPI:         stubOperationZoneAPI{},
+			operationGlobalAPI:       stubOperationGlobalAPI{},
+			instanceTemplateAPI:      stubInstanceTemplateAPI{},
+			instanceGroupManagersAPI: stubInstanceGroupManagersAPI{deleteErr: notFoundErr},
+		},
+		"templates not found": {
+			operationZoneAPI:         stubOperationZoneAPI{},
+			operationGlobalAPI:       stubOperationGlobalAPI{},
+			instanceTemplateAPI:      stubInstanceTemplateAPI{deleteErr: notFoundErr},
+			instanceGroupManagersAPI: stubInstanceGroupManagersAPI{},
+		},
 		"fail delete instanceGroupManager": {
 			operationZoneAPI:         stubOperationZoneAPI{},
 			operationGlobalAPI:       stubOperationGlobalAPI{},

cli/internal/gcp/client/network.go

@@ -65,6 +65,9 @@ func (c *Client) TerminateFirewall(ctx context.Context) error {
 			Project:  c.project,
 		}
 		resp, err := c.firewallsAPI.Delete(ctx, req)
+		if isNotFoundError(err) {
+			continue
+		}
 		if err != nil {
 			return err
 		}
@@ -130,28 +133,40 @@ func (c *Client) TerminateVPCs(ctx context.Context) error {
 		return err
 	}
 
-	var op Operation
+	if err := c.terminateVPC(ctx); err != nil {
-	var err error
-	if c.network != "" {
-		op, err = c.terminateVPC(ctx, c.network)
-		if err != nil {
 		return err
 	}
-		c.network = ""
-	}
 
-	return c.waitForOperations(ctx, []Operation{op})
+	return nil
 }
 
 // terminateVPC terminates a VPC network.
 //
 // If the network has firewall rules, these must be terminated first.
-func (c *Client) terminateVPC(ctx context.Context, network string) (Operation, error) {
+func (c *Client) terminateVPC(ctx context.Context) error {
+	if c.network == "" {
+		return nil
+	}
+
 	req := &computepb.DeleteNetworkRequest{
 		Project: c.project,
-		Network: network,
+		Network: c.network,
 	}
-	return c.networksAPI.Delete(ctx, req)
+	op, err := c.networksAPI.Delete(ctx, req)
+	if err != nil && !isNotFoundError(err) {
+		return err
+	}
+	if isNotFoundError(err) {
+		c.network = ""
+		return nil
+	}
+
+	if err := c.waitForOperations(ctx, []Operation{op}); err != nil {
+		return err
+	}
+
+	c.network = ""
+	return nil
 }
 
 func (c *Client) createSubnets(ctx context.Context) error {
@@ -189,16 +204,27 @@ func (c *Client) terminateSubnet(ctx context.Context) error {
 	if c.subnetwork == "" {
 		return nil
 	}
+
 	req := &computepb.DeleteSubnetworkRequest{
 		Project:    c.project,
 		Region:     c.region,
 		Subnetwork: c.subnetwork,
 	}
 	op, err := c.subnetworksAPI.Delete(ctx, req)
-	if err != nil {
+	if err != nil && !isNotFoundError(err) {
 		return err
 	}
-	return c.waitForOperations(ctx, []Operation{op})
+	if isNotFoundError(err) {
+		c.subnetwork = ""
+		return nil
+	}
+
+	if err := c.waitForOperations(ctx, []Operation{op}); err != nil {
+		return err
+	}
+
+	c.subnetwork = ""
+	return nil
 }
 
 // CreateLoadBalancer creates a load balancer.
@@ -305,33 +331,43 @@ func (c *Client) TerminateLoadBalancer(ctx context.Context) error {
 		Region:         c.region,
 		ForwardingRule: c.forwardingRule,
 	})
-	if err != nil {
+	if err != nil && !isNotFoundError(err) {
 		return err
 	}
+	if err == nil {
 		if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
 			return err
 		}
+	}
 
 	resp, err = c.backendServicesAPI.Delete(ctx, &computepb.DeleteRegionBackendServiceRequest{
 		Project:        c.project,
 		Region:         c.region,
 		BackendService: c.backendService,
 	})
-	if err != nil {
+	if err != nil && !isNotFoundError(err) {
 		return err
 	}
+	if err == nil {
 		if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
 			return err
 		}
+	}
 
 	resp, err = c.healthChecksAPI.Delete(ctx, &computepb.DeleteRegionHealthCheckRequest{
 		Project:     c.project,
 		Region:      c.region,
 		HealthCheck: c.healthCheck,
 	})
-	if err != nil {
+	if err != nil && !isNotFoundError(err) {
 		return err
 	}
 
-	return c.waitForOperations(ctx, []Operation{resp})
+	if err == nil {
+		if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }

cli/internal/gcp/client/network_test.go

@@ -3,10 +3,12 @@ package client
 import (
 	"context"
 	"errors"
+	"net/http"
 	"testing"
 
 	"github.com/edgelesssys/constellation/internal/cloud/cloudtypes"
 	"github.com/stretchr/testify/assert"
+	"google.golang.org/api/googleapi"
 	"google.golang.org/genproto/googleapis/cloud/compute/v1"
 	"google.golang.org/protobuf/proto"
 )
@@ -87,6 +89,8 @@ func TestCreateVPCs(t *testing.T) {
 
 func TestTerminateVPCs(t *testing.T) {
 	someErr := errors.New("failed")
+	notFoundErr := &googleapi.Error{Code: http.StatusNotFound}
+
 	testCases := map[string]struct {
 		operationGlobalAPI operationGlobalAPI
 		operationRegionAPI operationRegionAPI
@@ -94,6 +98,7 @@ func TestTerminateVPCs(t *testing.T) {
 		subnetworksAPI     subnetworksAPI
 		firewalls          []string
 		subnetwork         string
+		network            string
 		wantErr            bool
 	}{
 		"successful terminate": {
@@ -102,6 +107,7 @@ func TestTerminateVPCs(t *testing.T) {
 			networksAPI:        stubNetworksAPI{},
 			subnetworksAPI:     stubSubnetworksAPI{},
 			subnetwork:         "subnetwork-id-1",
+			network:            "network-id-1",
 		},
 		"subnetwork empty": {
 			operationGlobalAPI: stubOperationGlobalAPI{},
@@ -109,30 +115,58 @@ func TestTerminateVPCs(t *testing.T) {
 			networksAPI:        stubNetworksAPI{},
 			subnetworksAPI:     stubSubnetworksAPI{},
 			subnetwork:         "",
+			network:            "network-id-1",
+		},
+		"network empty": {
+			operationGlobalAPI: stubOperationGlobalAPI{},
+			operationRegionAPI: stubOperationRegionAPI{},
+			networksAPI:        stubNetworksAPI{},
+			subnetworksAPI:     stubSubnetworksAPI{},
+			subnetwork:         "subnetwork-id-1",
+			network:            "",
+		},
+		"subnetwork not found": {
+			operationGlobalAPI: stubOperationGlobalAPI{},
+			operationRegionAPI: stubOperationRegionAPI{},
+			networksAPI:        stubNetworksAPI{},
+			subnetworksAPI:     stubSubnetworksAPI{deleteErr: notFoundErr},
+			subnetwork:         "subnetwork-id-1",
+			network:            "network-id-1",
+		},
+		"network not found": {
+			operationGlobalAPI: stubOperationGlobalAPI{},
+			operationRegionAPI: stubOperationRegionAPI{},
+			networksAPI:        stubNetworksAPI{deleteErr: notFoundErr},
+			subnetworksAPI:     stubSubnetworksAPI{},
+			subnetwork:         "subnetwork-id-1",
+			network:            "network-id-1",
 		},
 		"failed wait global op": {
 			operationGlobalAPI: stubOperationGlobalAPI{waitErr: someErr},
 			operationRegionAPI: stubOperationRegionAPI{},
 			networksAPI:        stubNetworksAPI{},
 			subnetworksAPI:     stubSubnetworksAPI{},
-			wantErr:            true,
 			subnetwork:         "subnetwork-id-1",
+			network:            "network-id-1",
+			wantErr:            true,
 		},
 		"failed delete networks": {
 			operationGlobalAPI: stubOperationGlobalAPI{},
 			operationRegionAPI: stubOperationRegionAPI{},
 			networksAPI:        stubNetworksAPI{deleteErr: someErr},
 			subnetworksAPI:     stubSubnetworksAPI{},
-			wantErr:            true,
 			subnetwork:         "subnetwork-id-1",
+			network:            "network-id-1",
+			wantErr:            true,
 		},
 		"failed delete subnetworks": {
 			operationGlobalAPI: stubOperationGlobalAPI{},
 			operationRegionAPI: stubOperationRegionAPI{},
 			networksAPI:        stubNetworksAPI{},
 			subnetworksAPI:     stubSubnetworksAPI{deleteErr: someErr},
-			wantErr:            true,
 			subnetwork:         "subnetwork-id-1",
+			network:            "network-id-1",
+			wantErr:            true,
 		},
 		"must delete firewalls first": {
 			firewalls:          []string{"firewall-1", "firewall-2"},
@@ -140,8 +174,9 @@ func TestTerminateVPCs(t *testing.T) {
 			operationGlobalAPI: stubOperationGlobalAPI{},
 			networksAPI:        stubNetworksAPI{},
 			subnetworksAPI:     stubSubnetworksAPI{},
-			wantErr:            true,
 			subnetwork:         "subnetwork-id-1",
+			network:            "network-id-1",
+			wantErr:            true,
 		},
 	}
 
@@ -160,7 +195,7 @@ func TestTerminateVPCs(t *testing.T) {
 				networksAPI:        tc.networksAPI,
 				subnetworksAPI:     tc.subnetworksAPI,
 				firewalls:          tc.firewalls,
-				network:            "network-id-1",
+				network:            tc.network,
 				subnetwork:         tc.subnetwork,
 			}
 
@@ -169,6 +204,7 @@ func TestTerminateVPCs(t *testing.T) {
 			} else {
 				assert.NoError(client.TerminateVPCs(ctx))
 				assert.Empty(client.network)
+				assert.Empty(client.subnetwork)
 			}
 		})
 	}
@@ -254,6 +290,7 @@ func TestCreateFirewall(t *testing.T) {
 
 func TestTerminateFirewall(t *testing.T) {
 	someErr := errors.New("failed")
+	notFoundErr := &googleapi.Error{Code: http.StatusNotFound}
 
 	testCases := map[string]struct {
 		operationGlobalAPI operationGlobalAPI
@@ -271,6 +308,11 @@ func TestTerminateFirewall(t *testing.T) {
 			firewallsAPI:       stubFirewallsAPI{},
 			firewalls:          []string{},
 		},
+		"successful terminate when firewall not found": {
+			operationGlobalAPI: stubOperationGlobalAPI{},
+			firewallsAPI:       stubFirewallsAPI{deleteErr: notFoundErr},
+			firewalls:          []string{"firewall-1", "firewall-2"},
+		},
 		"failed to wait on global operation": {
 			operationGlobalAPI: stubOperationGlobalAPI{waitErr: someErr},
 			firewallsAPI:       stubFirewallsAPI{},
@@ -399,6 +441,8 @@ func TestCreateLoadBalancer(t *testing.T) {
 
 func TestTerminateLoadBalancer(t *testing.T) {
 	someErr := errors.New("failed")
+	notFoundErr := &googleapi.Error{Code: http.StatusNotFound}
+
 	testCases := map[string]struct {
 		operationRegionAPI operationRegionAPI
 		healthChecksAPI    healthChecksAPI
@@ -412,6 +456,24 @@ func TestTerminateLoadBalancer(t *testing.T) {
 			forwardingRulesAPI: stubForwardingRulesAPI{},
 			operationRegionAPI: stubOperationRegionAPI{},
 		},
+		"successful terminate when health check not found": {
+			healthChecksAPI:    stubHealthChecksAPI{deleteErr: notFoundErr},
+			backendServicesAPI: stubBackendServicesAPI{},
+			forwardingRulesAPI: stubForwardingRulesAPI{},
+			operationRegionAPI: stubOperationRegionAPI{},
+		},
+		"successful terminate when backend service not found": {
+			healthChecksAPI:    stubHealthChecksAPI{},
+			backendServicesAPI: stubBackendServicesAPI{deleteErr: notFoundErr},
+			forwardingRulesAPI: stubForwardingRulesAPI{},
+			operationRegionAPI: stubOperationRegionAPI{},
+		},
+		"successful terminate when forwarding rule not found": {
+			healthChecksAPI:    stubHealthChecksAPI{},
+			backendServicesAPI: stubBackendServicesAPI{},
+			forwardingRulesAPI: stubForwardingRulesAPI{deleteErr: notFoundErr},
+			operationRegionAPI: stubOperationRegionAPI{},
+		},
 		"TerminateLoadBalancer fails when deleting health check": {
 			healthChecksAPI:    stubHealthChecksAPI{deleteErr: someErr},
 			backendServicesAPI: stubBackendServicesAPI{},

cli/internal/gcp/client/serviceaccount.go

@@ -38,15 +38,18 @@ func (c *Client) CreateServiceAccount(ctx context.Context, input ServiceAccountI
 }
 
 func (c *Client) TerminateServiceAccount(ctx context.Context) error {
-	if c.serviceAccount != "" {
+	if c.serviceAccount == "" {
+		return nil
+	}
+
 	req := &adminpb.DeleteServiceAccountRequest{
 		Name: "projects/-/serviceAccounts/" + c.serviceAccount,
 	}
-		if err := c.iamAPI.DeleteServiceAccount(ctx, req); err != nil {
+	if err := c.iamAPI.DeleteServiceAccount(ctx, req); err != nil && !isNotFoundError(err) {
 		return fmt.Errorf("deleting service account: %w", err)
 	}
+
 	c.serviceAccount = ""
-	}
 	return nil
 }