Check for 404 errors in GCP termination

This commit is contained in:
katexochen 2022-08-01 14:58:23 +02:00 committed by Paul Meyer
parent 9f599c3993
commit c954ec089f
6 changed files with 179 additions and 43 deletions

View File

@ -4,8 +4,10 @@ import (
"context"
"crypto/rand"
"encoding/json"
"errors"
"fmt"
"math/big"
"net/http"
compute "cloud.google.com/go/compute/apiv1"
admin "cloud.google.com/go/iam/admin/apiv1"
@ -15,6 +17,7 @@ import (
"github.com/edgelesssys/constellation/internal/state"
"go.uber.org/multierr"
"golang.org/x/oauth2/google"
"google.golang.org/api/googleapi"
)
// Client is a client for the Google Compute Engine.
@ -318,3 +321,11 @@ func closeAll(closers []closer) error {
}
return err
}
func isNotFoundError(err error) bool {
var gAPIErr *googleapi.Error
if !errors.As(err, &gAPIErr) {
return false
}
return gAPIErr.Code == http.StatusNotFound
}

View File

@ -131,20 +131,24 @@ func (c *Client) TerminateInstances(ctx context.Context) error {
ops := []Operation{}
if c.workerInstanceGroup != "" {
op, err := c.deleteInstanceGroupManager(ctx, c.workerInstanceGroup)
if err != nil {
if err != nil && !isNotFoundError(err) {
return fmt.Errorf("deleting instanceGroupManager '%s': %w", c.workerInstanceGroup, err)
}
ops = append(ops, op)
if err == nil {
ops = append(ops, op)
}
c.workerInstanceGroup = ""
c.workers = make(cloudtypes.Instances)
}
if c.controlPlaneInstanceGroup != "" {
op, err := c.deleteInstanceGroupManager(ctx, c.controlPlaneInstanceGroup)
if err != nil {
if err != nil && !isNotFoundError(err) {
return fmt.Errorf("deleting instanceGroupManager '%s': %w", c.controlPlaneInstanceGroup, err)
}
ops = append(ops, op)
if err == nil {
ops = append(ops, op)
}
c.controlPlaneInstanceGroup = ""
c.controlPlanes = make(cloudtypes.Instances)
}
@ -155,18 +159,22 @@ func (c *Client) TerminateInstances(ctx context.Context) error {
if c.workerTemplate != "" {
op, err := c.deleteInstanceTemplate(ctx, c.workerTemplate)
if err != nil {
if err != nil && !isNotFoundError(err) {
return fmt.Errorf("deleting instanceTemplate: %w", err)
}
ops = append(ops, op)
if err == nil {
ops = append(ops, op)
}
c.workerTemplate = ""
}
if c.controlPlaneTemplate != "" {
op, err := c.deleteInstanceTemplate(ctx, c.controlPlaneTemplate)
if err != nil {
if err != nil && !isNotFoundError(err) {
return fmt.Errorf("deleting instanceTemplate: %w", err)
}
ops = append(ops, op)
if err == nil {
ops = append(ops, op)
}
c.controlPlaneTemplate = ""
}
return c.waitForOperations(ctx, ops)

View File

@ -3,10 +3,12 @@ package client
import (
"context"
"errors"
"net/http"
"testing"
"github.com/edgelesssys/constellation/internal/cloud/cloudtypes"
"github.com/stretchr/testify/assert"
"google.golang.org/api/googleapi"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
"google.golang.org/protobuf/proto"
)
@ -180,6 +182,8 @@ func TestCreateInstances(t *testing.T) {
func TestTerminateInstances(t *testing.T) {
someErr := errors.New("failed")
notFoundErr := &googleapi.Error{Code: http.StatusNotFound}
testCases := map[string]struct {
operationZoneAPI operationZoneAPI
operationGlobalAPI operationGlobalAPI
@ -202,6 +206,18 @@ func TestTerminateInstances(t *testing.T) {
instanceGroupManagersAPI: stubInstanceGroupManagersAPI{},
missingWorkerInstanceGroup: true,
},
"instances not found": {
operationZoneAPI: stubOperationZoneAPI{},
operationGlobalAPI: stubOperationGlobalAPI{},
instanceTemplateAPI: stubInstanceTemplateAPI{},
instanceGroupManagersAPI: stubInstanceGroupManagersAPI{deleteErr: notFoundErr},
},
"templates not found": {
operationZoneAPI: stubOperationZoneAPI{},
operationGlobalAPI: stubOperationGlobalAPI{},
instanceTemplateAPI: stubInstanceTemplateAPI{deleteErr: notFoundErr},
instanceGroupManagersAPI: stubInstanceGroupManagersAPI{},
},
"fail delete instanceGroupManager": {
operationZoneAPI: stubOperationZoneAPI{},
operationGlobalAPI: stubOperationGlobalAPI{},

View File

@ -65,6 +65,9 @@ func (c *Client) TerminateFirewall(ctx context.Context) error {
Project: c.project,
}
resp, err := c.firewallsAPI.Delete(ctx, req)
if isNotFoundError(err) {
continue
}
if err != nil {
return err
}
@ -130,28 +133,40 @@ func (c *Client) TerminateVPCs(ctx context.Context) error {
return err
}
var op Operation
var err error
if c.network != "" {
op, err = c.terminateVPC(ctx, c.network)
if err != nil {
return err
}
c.network = ""
if err := c.terminateVPC(ctx); err != nil {
return err
}
return c.waitForOperations(ctx, []Operation{op})
return nil
}
// terminateVPC terminates a VPC network.
//
// If the network has firewall rules, these must be terminated first.
func (c *Client) terminateVPC(ctx context.Context, network string) (Operation, error) {
func (c *Client) terminateVPC(ctx context.Context) error {
if c.network == "" {
return nil
}
req := &computepb.DeleteNetworkRequest{
Project: c.project,
Network: network,
Network: c.network,
}
return c.networksAPI.Delete(ctx, req)
op, err := c.networksAPI.Delete(ctx, req)
if err != nil && !isNotFoundError(err) {
return err
}
if isNotFoundError(err) {
c.network = ""
return nil
}
if err := c.waitForOperations(ctx, []Operation{op}); err != nil {
return err
}
c.network = ""
return nil
}
func (c *Client) createSubnets(ctx context.Context) error {
@ -189,16 +204,27 @@ func (c *Client) terminateSubnet(ctx context.Context) error {
if c.subnetwork == "" {
return nil
}
req := &computepb.DeleteSubnetworkRequest{
Project: c.project,
Region: c.region,
Subnetwork: c.subnetwork,
}
op, err := c.subnetworksAPI.Delete(ctx, req)
if err != nil {
if err != nil && !isNotFoundError(err) {
return err
}
return c.waitForOperations(ctx, []Operation{op})
if isNotFoundError(err) {
c.subnetwork = ""
return nil
}
if err := c.waitForOperations(ctx, []Operation{op}); err != nil {
return err
}
c.subnetwork = ""
return nil
}
// CreateLoadBalancer creates a load balancer.
@ -305,11 +331,13 @@ func (c *Client) TerminateLoadBalancer(ctx context.Context) error {
Region: c.region,
ForwardingRule: c.forwardingRule,
})
if err != nil {
if err != nil && !isNotFoundError(err) {
return err
}
if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
return err
if err == nil {
if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
return err
}
}
resp, err = c.backendServicesAPI.Delete(ctx, &computepb.DeleteRegionBackendServiceRequest{
@ -317,11 +345,13 @@ func (c *Client) TerminateLoadBalancer(ctx context.Context) error {
Region: c.region,
BackendService: c.backendService,
})
if err != nil {
if err != nil && !isNotFoundError(err) {
return err
}
if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
return err
if err == nil {
if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
return err
}
}
resp, err = c.healthChecksAPI.Delete(ctx, &computepb.DeleteRegionHealthCheckRequest{
@ -329,9 +359,15 @@ func (c *Client) TerminateLoadBalancer(ctx context.Context) error {
Region: c.region,
HealthCheck: c.healthCheck,
})
if err != nil {
if err != nil && !isNotFoundError(err) {
return err
}
return c.waitForOperations(ctx, []Operation{resp})
if err == nil {
if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
return err
}
}
return nil
}

View File

@ -3,10 +3,12 @@ package client
import (
"context"
"errors"
"net/http"
"testing"
"github.com/edgelesssys/constellation/internal/cloud/cloudtypes"
"github.com/stretchr/testify/assert"
"google.golang.org/api/googleapi"
"google.golang.org/genproto/googleapis/cloud/compute/v1"
"google.golang.org/protobuf/proto"
)
@ -87,6 +89,8 @@ func TestCreateVPCs(t *testing.T) {
func TestTerminateVPCs(t *testing.T) {
someErr := errors.New("failed")
notFoundErr := &googleapi.Error{Code: http.StatusNotFound}
testCases := map[string]struct {
operationGlobalAPI operationGlobalAPI
operationRegionAPI operationRegionAPI
@ -94,6 +98,7 @@ func TestTerminateVPCs(t *testing.T) {
subnetworksAPI subnetworksAPI
firewalls []string
subnetwork string
network string
wantErr bool
}{
"successful terminate": {
@ -102,6 +107,7 @@ func TestTerminateVPCs(t *testing.T) {
networksAPI: stubNetworksAPI{},
subnetworksAPI: stubSubnetworksAPI{},
subnetwork: "subnetwork-id-1",
network: "network-id-1",
},
"subnetwork empty": {
operationGlobalAPI: stubOperationGlobalAPI{},
@ -109,30 +115,58 @@ func TestTerminateVPCs(t *testing.T) {
networksAPI: stubNetworksAPI{},
subnetworksAPI: stubSubnetworksAPI{},
subnetwork: "",
network: "network-id-1",
},
"network empty": {
operationGlobalAPI: stubOperationGlobalAPI{},
operationRegionAPI: stubOperationRegionAPI{},
networksAPI: stubNetworksAPI{},
subnetworksAPI: stubSubnetworksAPI{},
subnetwork: "subnetwork-id-1",
network: "",
},
"subnetwork not found": {
operationGlobalAPI: stubOperationGlobalAPI{},
operationRegionAPI: stubOperationRegionAPI{},
networksAPI: stubNetworksAPI{},
subnetworksAPI: stubSubnetworksAPI{deleteErr: notFoundErr},
subnetwork: "subnetwork-id-1",
network: "network-id-1",
},
"network not found": {
operationGlobalAPI: stubOperationGlobalAPI{},
operationRegionAPI: stubOperationRegionAPI{},
networksAPI: stubNetworksAPI{deleteErr: notFoundErr},
subnetworksAPI: stubSubnetworksAPI{},
subnetwork: "subnetwork-id-1",
network: "network-id-1",
},
"failed wait global op": {
operationGlobalAPI: stubOperationGlobalAPI{waitErr: someErr},
operationRegionAPI: stubOperationRegionAPI{},
networksAPI: stubNetworksAPI{},
subnetworksAPI: stubSubnetworksAPI{},
wantErr: true,
subnetwork: "subnetwork-id-1",
network: "network-id-1",
wantErr: true,
},
"failed delete networks": {
operationGlobalAPI: stubOperationGlobalAPI{},
operationRegionAPI: stubOperationRegionAPI{},
networksAPI: stubNetworksAPI{deleteErr: someErr},
subnetworksAPI: stubSubnetworksAPI{},
wantErr: true,
subnetwork: "subnetwork-id-1",
network: "network-id-1",
wantErr: true,
},
"failed delete subnetworks": {
operationGlobalAPI: stubOperationGlobalAPI{},
operationRegionAPI: stubOperationRegionAPI{},
networksAPI: stubNetworksAPI{},
subnetworksAPI: stubSubnetworksAPI{deleteErr: someErr},
wantErr: true,
subnetwork: "subnetwork-id-1",
network: "network-id-1",
wantErr: true,
},
"must delete firewalls first": {
firewalls: []string{"firewall-1", "firewall-2"},
@ -140,8 +174,9 @@ func TestTerminateVPCs(t *testing.T) {
operationGlobalAPI: stubOperationGlobalAPI{},
networksAPI: stubNetworksAPI{},
subnetworksAPI: stubSubnetworksAPI{},
wantErr: true,
subnetwork: "subnetwork-id-1",
network: "network-id-1",
wantErr: true,
},
}
@ -160,7 +195,7 @@ func TestTerminateVPCs(t *testing.T) {
networksAPI: tc.networksAPI,
subnetworksAPI: tc.subnetworksAPI,
firewalls: tc.firewalls,
network: "network-id-1",
network: tc.network,
subnetwork: tc.subnetwork,
}
@ -169,6 +204,7 @@ func TestTerminateVPCs(t *testing.T) {
} else {
assert.NoError(client.TerminateVPCs(ctx))
assert.Empty(client.network)
assert.Empty(client.subnetwork)
}
})
}
@ -254,6 +290,7 @@ func TestCreateFirewall(t *testing.T) {
func TestTerminateFirewall(t *testing.T) {
someErr := errors.New("failed")
notFoundErr := &googleapi.Error{Code: http.StatusNotFound}
testCases := map[string]struct {
operationGlobalAPI operationGlobalAPI
@ -271,6 +308,11 @@ func TestTerminateFirewall(t *testing.T) {
firewallsAPI: stubFirewallsAPI{},
firewalls: []string{},
},
"successful terminate when firewall not found": {
operationGlobalAPI: stubOperationGlobalAPI{},
firewallsAPI: stubFirewallsAPI{deleteErr: notFoundErr},
firewalls: []string{"firewall-1", "firewall-2"},
},
"failed to wait on global operation": {
operationGlobalAPI: stubOperationGlobalAPI{waitErr: someErr},
firewallsAPI: stubFirewallsAPI{},
@ -399,6 +441,8 @@ func TestCreateLoadBalancer(t *testing.T) {
func TestTerminateLoadBalancer(t *testing.T) {
someErr := errors.New("failed")
notFoundErr := &googleapi.Error{Code: http.StatusNotFound}
testCases := map[string]struct {
operationRegionAPI operationRegionAPI
healthChecksAPI healthChecksAPI
@ -412,6 +456,24 @@ func TestTerminateLoadBalancer(t *testing.T) {
forwardingRulesAPI: stubForwardingRulesAPI{},
operationRegionAPI: stubOperationRegionAPI{},
},
"successful terminate when health check not found": {
healthChecksAPI: stubHealthChecksAPI{deleteErr: notFoundErr},
backendServicesAPI: stubBackendServicesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{},
operationRegionAPI: stubOperationRegionAPI{},
},
"successful terminate when backend service not found": {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{deleteErr: notFoundErr},
forwardingRulesAPI: stubForwardingRulesAPI{},
operationRegionAPI: stubOperationRegionAPI{},
},
"successful terminate when forwarding rule not found": {
healthChecksAPI: stubHealthChecksAPI{},
backendServicesAPI: stubBackendServicesAPI{},
forwardingRulesAPI: stubForwardingRulesAPI{deleteErr: notFoundErr},
operationRegionAPI: stubOperationRegionAPI{},
},
"TerminateLoadBalancer fails when deleting health check": {
healthChecksAPI: stubHealthChecksAPI{deleteErr: someErr},
backendServicesAPI: stubBackendServicesAPI{},

View File

@ -38,15 +38,18 @@ func (c *Client) CreateServiceAccount(ctx context.Context, input ServiceAccountI
}
func (c *Client) TerminateServiceAccount(ctx context.Context) error {
if c.serviceAccount != "" {
req := &adminpb.DeleteServiceAccountRequest{
Name: "projects/-/serviceAccounts/" + c.serviceAccount,
}
if err := c.iamAPI.DeleteServiceAccount(ctx, req); err != nil {
return fmt.Errorf("deleting service account: %w", err)
}
c.serviceAccount = ""
if c.serviceAccount == "" {
return nil
}
req := &adminpb.DeleteServiceAccountRequest{
Name: "projects/-/serviceAccounts/" + c.serviceAccount,
}
if err := c.iamAPI.DeleteServiceAccount(ctx, req); err != nil && !isNotFoundError(err) {
return fmt.Errorf("deleting service account: %w", err)
}
c.serviceAccount = ""
return nil
}