mirror of
https://github.com/edgelesssys/constellation.git
synced 2024-10-01 01:36:09 -04:00
Check for 404 errors in GCP termination
This commit is contained in:
parent
9f599c3993
commit
c954ec089f
@ -4,8 +4,10 @@ import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net/http"
|
||||
|
||||
compute "cloud.google.com/go/compute/apiv1"
|
||||
admin "cloud.google.com/go/iam/admin/apiv1"
|
||||
@ -15,6 +17,7 @@ import (
|
||||
"github.com/edgelesssys/constellation/internal/state"
|
||||
"go.uber.org/multierr"
|
||||
"golang.org/x/oauth2/google"
|
||||
"google.golang.org/api/googleapi"
|
||||
)
|
||||
|
||||
// Client is a client for the Google Compute Engine.
|
||||
@ -318,3 +321,11 @@ func closeAll(closers []closer) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func isNotFoundError(err error) bool {
|
||||
var gAPIErr *googleapi.Error
|
||||
if !errors.As(err, &gAPIErr) {
|
||||
return false
|
||||
}
|
||||
return gAPIErr.Code == http.StatusNotFound
|
||||
}
|
||||
|
@ -131,20 +131,24 @@ func (c *Client) TerminateInstances(ctx context.Context) error {
|
||||
ops := []Operation{}
|
||||
if c.workerInstanceGroup != "" {
|
||||
op, err := c.deleteInstanceGroupManager(ctx, c.workerInstanceGroup)
|
||||
if err != nil {
|
||||
if err != nil && !isNotFoundError(err) {
|
||||
return fmt.Errorf("deleting instanceGroupManager '%s': %w", c.workerInstanceGroup, err)
|
||||
}
|
||||
ops = append(ops, op)
|
||||
if err == nil {
|
||||
ops = append(ops, op)
|
||||
}
|
||||
c.workerInstanceGroup = ""
|
||||
c.workers = make(cloudtypes.Instances)
|
||||
}
|
||||
|
||||
if c.controlPlaneInstanceGroup != "" {
|
||||
op, err := c.deleteInstanceGroupManager(ctx, c.controlPlaneInstanceGroup)
|
||||
if err != nil {
|
||||
if err != nil && !isNotFoundError(err) {
|
||||
return fmt.Errorf("deleting instanceGroupManager '%s': %w", c.controlPlaneInstanceGroup, err)
|
||||
}
|
||||
ops = append(ops, op)
|
||||
if err == nil {
|
||||
ops = append(ops, op)
|
||||
}
|
||||
c.controlPlaneInstanceGroup = ""
|
||||
c.controlPlanes = make(cloudtypes.Instances)
|
||||
}
|
||||
@ -155,18 +159,22 @@ func (c *Client) TerminateInstances(ctx context.Context) error {
|
||||
|
||||
if c.workerTemplate != "" {
|
||||
op, err := c.deleteInstanceTemplate(ctx, c.workerTemplate)
|
||||
if err != nil {
|
||||
if err != nil && !isNotFoundError(err) {
|
||||
return fmt.Errorf("deleting instanceTemplate: %w", err)
|
||||
}
|
||||
ops = append(ops, op)
|
||||
if err == nil {
|
||||
ops = append(ops, op)
|
||||
}
|
||||
c.workerTemplate = ""
|
||||
}
|
||||
if c.controlPlaneTemplate != "" {
|
||||
op, err := c.deleteInstanceTemplate(ctx, c.controlPlaneTemplate)
|
||||
if err != nil {
|
||||
if err != nil && !isNotFoundError(err) {
|
||||
return fmt.Errorf("deleting instanceTemplate: %w", err)
|
||||
}
|
||||
ops = append(ops, op)
|
||||
if err == nil {
|
||||
ops = append(ops, op)
|
||||
}
|
||||
c.controlPlaneTemplate = ""
|
||||
}
|
||||
return c.waitForOperations(ctx, ops)
|
||||
|
@ -3,10 +3,12 @@ package client
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/edgelesssys/constellation/internal/cloud/cloudtypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/api/googleapi"
|
||||
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
@ -180,6 +182,8 @@ func TestCreateInstances(t *testing.T) {
|
||||
|
||||
func TestTerminateInstances(t *testing.T) {
|
||||
someErr := errors.New("failed")
|
||||
notFoundErr := &googleapi.Error{Code: http.StatusNotFound}
|
||||
|
||||
testCases := map[string]struct {
|
||||
operationZoneAPI operationZoneAPI
|
||||
operationGlobalAPI operationGlobalAPI
|
||||
@ -202,6 +206,18 @@ func TestTerminateInstances(t *testing.T) {
|
||||
instanceGroupManagersAPI: stubInstanceGroupManagersAPI{},
|
||||
missingWorkerInstanceGroup: true,
|
||||
},
|
||||
"instances not found": {
|
||||
operationZoneAPI: stubOperationZoneAPI{},
|
||||
operationGlobalAPI: stubOperationGlobalAPI{},
|
||||
instanceTemplateAPI: stubInstanceTemplateAPI{},
|
||||
instanceGroupManagersAPI: stubInstanceGroupManagersAPI{deleteErr: notFoundErr},
|
||||
},
|
||||
"templates not found": {
|
||||
operationZoneAPI: stubOperationZoneAPI{},
|
||||
operationGlobalAPI: stubOperationGlobalAPI{},
|
||||
instanceTemplateAPI: stubInstanceTemplateAPI{deleteErr: notFoundErr},
|
||||
instanceGroupManagersAPI: stubInstanceGroupManagersAPI{},
|
||||
},
|
||||
"fail delete instanceGroupManager": {
|
||||
operationZoneAPI: stubOperationZoneAPI{},
|
||||
operationGlobalAPI: stubOperationGlobalAPI{},
|
||||
|
@ -65,6 +65,9 @@ func (c *Client) TerminateFirewall(ctx context.Context) error {
|
||||
Project: c.project,
|
||||
}
|
||||
resp, err := c.firewallsAPI.Delete(ctx, req)
|
||||
if isNotFoundError(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -130,28 +133,40 @@ func (c *Client) TerminateVPCs(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
var op Operation
|
||||
var err error
|
||||
if c.network != "" {
|
||||
op, err = c.terminateVPC(ctx, c.network)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.network = ""
|
||||
if err := c.terminateVPC(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return c.waitForOperations(ctx, []Operation{op})
|
||||
return nil
|
||||
}
|
||||
|
||||
// terminateVPC terminates a VPC network.
|
||||
//
|
||||
// If the network has firewall rules, these must be terminated first.
|
||||
func (c *Client) terminateVPC(ctx context.Context, network string) (Operation, error) {
|
||||
func (c *Client) terminateVPC(ctx context.Context) error {
|
||||
if c.network == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
req := &computepb.DeleteNetworkRequest{
|
||||
Project: c.project,
|
||||
Network: network,
|
||||
Network: c.network,
|
||||
}
|
||||
return c.networksAPI.Delete(ctx, req)
|
||||
op, err := c.networksAPI.Delete(ctx, req)
|
||||
if err != nil && !isNotFoundError(err) {
|
||||
return err
|
||||
}
|
||||
if isNotFoundError(err) {
|
||||
c.network = ""
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := c.waitForOperations(ctx, []Operation{op}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.network = ""
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) createSubnets(ctx context.Context) error {
|
||||
@ -189,16 +204,27 @@ func (c *Client) terminateSubnet(ctx context.Context) error {
|
||||
if c.subnetwork == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
req := &computepb.DeleteSubnetworkRequest{
|
||||
Project: c.project,
|
||||
Region: c.region,
|
||||
Subnetwork: c.subnetwork,
|
||||
}
|
||||
op, err := c.subnetworksAPI.Delete(ctx, req)
|
||||
if err != nil {
|
||||
if err != nil && !isNotFoundError(err) {
|
||||
return err
|
||||
}
|
||||
return c.waitForOperations(ctx, []Operation{op})
|
||||
if isNotFoundError(err) {
|
||||
c.subnetwork = ""
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := c.waitForOperations(ctx, []Operation{op}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.subnetwork = ""
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateLoadBalancer creates a load balancer.
|
||||
@ -305,11 +331,13 @@ func (c *Client) TerminateLoadBalancer(ctx context.Context) error {
|
||||
Region: c.region,
|
||||
ForwardingRule: c.forwardingRule,
|
||||
})
|
||||
if err != nil {
|
||||
if err != nil && !isNotFoundError(err) {
|
||||
return err
|
||||
}
|
||||
if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
|
||||
return err
|
||||
if err == nil {
|
||||
if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
resp, err = c.backendServicesAPI.Delete(ctx, &computepb.DeleteRegionBackendServiceRequest{
|
||||
@ -317,11 +345,13 @@ func (c *Client) TerminateLoadBalancer(ctx context.Context) error {
|
||||
Region: c.region,
|
||||
BackendService: c.backendService,
|
||||
})
|
||||
if err != nil {
|
||||
if err != nil && !isNotFoundError(err) {
|
||||
return err
|
||||
}
|
||||
if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
|
||||
return err
|
||||
if err == nil {
|
||||
if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
resp, err = c.healthChecksAPI.Delete(ctx, &computepb.DeleteRegionHealthCheckRequest{
|
||||
@ -329,9 +359,15 @@ func (c *Client) TerminateLoadBalancer(ctx context.Context) error {
|
||||
Region: c.region,
|
||||
HealthCheck: c.healthCheck,
|
||||
})
|
||||
if err != nil {
|
||||
if err != nil && !isNotFoundError(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
return c.waitForOperations(ctx, []Operation{resp})
|
||||
if err == nil {
|
||||
if err := c.waitForOperations(ctx, []Operation{resp}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -3,10 +3,12 @@ package client
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/edgelesssys/constellation/internal/cloud/cloudtypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/genproto/googleapis/cloud/compute/v1"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
@ -87,6 +89,8 @@ func TestCreateVPCs(t *testing.T) {
|
||||
|
||||
func TestTerminateVPCs(t *testing.T) {
|
||||
someErr := errors.New("failed")
|
||||
notFoundErr := &googleapi.Error{Code: http.StatusNotFound}
|
||||
|
||||
testCases := map[string]struct {
|
||||
operationGlobalAPI operationGlobalAPI
|
||||
operationRegionAPI operationRegionAPI
|
||||
@ -94,6 +98,7 @@ func TestTerminateVPCs(t *testing.T) {
|
||||
subnetworksAPI subnetworksAPI
|
||||
firewalls []string
|
||||
subnetwork string
|
||||
network string
|
||||
wantErr bool
|
||||
}{
|
||||
"successful terminate": {
|
||||
@ -102,6 +107,7 @@ func TestTerminateVPCs(t *testing.T) {
|
||||
networksAPI: stubNetworksAPI{},
|
||||
subnetworksAPI: stubSubnetworksAPI{},
|
||||
subnetwork: "subnetwork-id-1",
|
||||
network: "network-id-1",
|
||||
},
|
||||
"subnetwork empty": {
|
||||
operationGlobalAPI: stubOperationGlobalAPI{},
|
||||
@ -109,30 +115,58 @@ func TestTerminateVPCs(t *testing.T) {
|
||||
networksAPI: stubNetworksAPI{},
|
||||
subnetworksAPI: stubSubnetworksAPI{},
|
||||
subnetwork: "",
|
||||
network: "network-id-1",
|
||||
},
|
||||
"network empty": {
|
||||
operationGlobalAPI: stubOperationGlobalAPI{},
|
||||
operationRegionAPI: stubOperationRegionAPI{},
|
||||
networksAPI: stubNetworksAPI{},
|
||||
subnetworksAPI: stubSubnetworksAPI{},
|
||||
subnetwork: "subnetwork-id-1",
|
||||
network: "",
|
||||
},
|
||||
"subnetwork not found": {
|
||||
operationGlobalAPI: stubOperationGlobalAPI{},
|
||||
operationRegionAPI: stubOperationRegionAPI{},
|
||||
networksAPI: stubNetworksAPI{},
|
||||
subnetworksAPI: stubSubnetworksAPI{deleteErr: notFoundErr},
|
||||
subnetwork: "subnetwork-id-1",
|
||||
network: "network-id-1",
|
||||
},
|
||||
"network not found": {
|
||||
operationGlobalAPI: stubOperationGlobalAPI{},
|
||||
operationRegionAPI: stubOperationRegionAPI{},
|
||||
networksAPI: stubNetworksAPI{deleteErr: notFoundErr},
|
||||
subnetworksAPI: stubSubnetworksAPI{},
|
||||
subnetwork: "subnetwork-id-1",
|
||||
network: "network-id-1",
|
||||
},
|
||||
"failed wait global op": {
|
||||
operationGlobalAPI: stubOperationGlobalAPI{waitErr: someErr},
|
||||
operationRegionAPI: stubOperationRegionAPI{},
|
||||
networksAPI: stubNetworksAPI{},
|
||||
subnetworksAPI: stubSubnetworksAPI{},
|
||||
wantErr: true,
|
||||
subnetwork: "subnetwork-id-1",
|
||||
network: "network-id-1",
|
||||
wantErr: true,
|
||||
},
|
||||
"failed delete networks": {
|
||||
operationGlobalAPI: stubOperationGlobalAPI{},
|
||||
operationRegionAPI: stubOperationRegionAPI{},
|
||||
networksAPI: stubNetworksAPI{deleteErr: someErr},
|
||||
subnetworksAPI: stubSubnetworksAPI{},
|
||||
wantErr: true,
|
||||
subnetwork: "subnetwork-id-1",
|
||||
network: "network-id-1",
|
||||
wantErr: true,
|
||||
},
|
||||
"failed delete subnetworks": {
|
||||
operationGlobalAPI: stubOperationGlobalAPI{},
|
||||
operationRegionAPI: stubOperationRegionAPI{},
|
||||
networksAPI: stubNetworksAPI{},
|
||||
subnetworksAPI: stubSubnetworksAPI{deleteErr: someErr},
|
||||
wantErr: true,
|
||||
subnetwork: "subnetwork-id-1",
|
||||
network: "network-id-1",
|
||||
wantErr: true,
|
||||
},
|
||||
"must delete firewalls first": {
|
||||
firewalls: []string{"firewall-1", "firewall-2"},
|
||||
@ -140,8 +174,9 @@ func TestTerminateVPCs(t *testing.T) {
|
||||
operationGlobalAPI: stubOperationGlobalAPI{},
|
||||
networksAPI: stubNetworksAPI{},
|
||||
subnetworksAPI: stubSubnetworksAPI{},
|
||||
wantErr: true,
|
||||
subnetwork: "subnetwork-id-1",
|
||||
network: "network-id-1",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
@ -160,7 +195,7 @@ func TestTerminateVPCs(t *testing.T) {
|
||||
networksAPI: tc.networksAPI,
|
||||
subnetworksAPI: tc.subnetworksAPI,
|
||||
firewalls: tc.firewalls,
|
||||
network: "network-id-1",
|
||||
network: tc.network,
|
||||
subnetwork: tc.subnetwork,
|
||||
}
|
||||
|
||||
@ -169,6 +204,7 @@ func TestTerminateVPCs(t *testing.T) {
|
||||
} else {
|
||||
assert.NoError(client.TerminateVPCs(ctx))
|
||||
assert.Empty(client.network)
|
||||
assert.Empty(client.subnetwork)
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -254,6 +290,7 @@ func TestCreateFirewall(t *testing.T) {
|
||||
|
||||
func TestTerminateFirewall(t *testing.T) {
|
||||
someErr := errors.New("failed")
|
||||
notFoundErr := &googleapi.Error{Code: http.StatusNotFound}
|
||||
|
||||
testCases := map[string]struct {
|
||||
operationGlobalAPI operationGlobalAPI
|
||||
@ -271,6 +308,11 @@ func TestTerminateFirewall(t *testing.T) {
|
||||
firewallsAPI: stubFirewallsAPI{},
|
||||
firewalls: []string{},
|
||||
},
|
||||
"successful terminate when firewall not found": {
|
||||
operationGlobalAPI: stubOperationGlobalAPI{},
|
||||
firewallsAPI: stubFirewallsAPI{deleteErr: notFoundErr},
|
||||
firewalls: []string{"firewall-1", "firewall-2"},
|
||||
},
|
||||
"failed to wait on global operation": {
|
||||
operationGlobalAPI: stubOperationGlobalAPI{waitErr: someErr},
|
||||
firewallsAPI: stubFirewallsAPI{},
|
||||
@ -399,6 +441,8 @@ func TestCreateLoadBalancer(t *testing.T) {
|
||||
|
||||
func TestTerminateLoadBalancer(t *testing.T) {
|
||||
someErr := errors.New("failed")
|
||||
notFoundErr := &googleapi.Error{Code: http.StatusNotFound}
|
||||
|
||||
testCases := map[string]struct {
|
||||
operationRegionAPI operationRegionAPI
|
||||
healthChecksAPI healthChecksAPI
|
||||
@ -412,6 +456,24 @@ func TestTerminateLoadBalancer(t *testing.T) {
|
||||
forwardingRulesAPI: stubForwardingRulesAPI{},
|
||||
operationRegionAPI: stubOperationRegionAPI{},
|
||||
},
|
||||
"successful terminate when health check not found": {
|
||||
healthChecksAPI: stubHealthChecksAPI{deleteErr: notFoundErr},
|
||||
backendServicesAPI: stubBackendServicesAPI{},
|
||||
forwardingRulesAPI: stubForwardingRulesAPI{},
|
||||
operationRegionAPI: stubOperationRegionAPI{},
|
||||
},
|
||||
"successful terminate when backend service not found": {
|
||||
healthChecksAPI: stubHealthChecksAPI{},
|
||||
backendServicesAPI: stubBackendServicesAPI{deleteErr: notFoundErr},
|
||||
forwardingRulesAPI: stubForwardingRulesAPI{},
|
||||
operationRegionAPI: stubOperationRegionAPI{},
|
||||
},
|
||||
"successful terminate when forwarding rule not found": {
|
||||
healthChecksAPI: stubHealthChecksAPI{},
|
||||
backendServicesAPI: stubBackendServicesAPI{},
|
||||
forwardingRulesAPI: stubForwardingRulesAPI{deleteErr: notFoundErr},
|
||||
operationRegionAPI: stubOperationRegionAPI{},
|
||||
},
|
||||
"TerminateLoadBalancer fails when deleting health check": {
|
||||
healthChecksAPI: stubHealthChecksAPI{deleteErr: someErr},
|
||||
backendServicesAPI: stubBackendServicesAPI{},
|
||||
|
@ -38,15 +38,18 @@ func (c *Client) CreateServiceAccount(ctx context.Context, input ServiceAccountI
|
||||
}
|
||||
|
||||
func (c *Client) TerminateServiceAccount(ctx context.Context) error {
|
||||
if c.serviceAccount != "" {
|
||||
req := &adminpb.DeleteServiceAccountRequest{
|
||||
Name: "projects/-/serviceAccounts/" + c.serviceAccount,
|
||||
}
|
||||
if err := c.iamAPI.DeleteServiceAccount(ctx, req); err != nil {
|
||||
return fmt.Errorf("deleting service account: %w", err)
|
||||
}
|
||||
c.serviceAccount = ""
|
||||
if c.serviceAccount == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
req := &adminpb.DeleteServiceAccountRequest{
|
||||
Name: "projects/-/serviceAccounts/" + c.serviceAccount,
|
||||
}
|
||||
if err := c.iamAPI.DeleteServiceAccount(ctx, req); err != nil && !isNotFoundError(err) {
|
||||
return fmt.Errorf("deleting service account: %w", err)
|
||||
}
|
||||
|
||||
c.serviceAccount = ""
|
||||
return nil
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user