From e13f4d84c3037ac4b06faea807db696dcfb88c3a Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Thu, 9 Jun 2022 22:26:36 +0200 Subject: [PATCH] add gcp loadbalancer --- CHANGELOG.md | 1 + cli/internal/cloudcmd/clients.go | 2 + cli/internal/cloudcmd/clients_test.go | 35 ++++ cli/internal/cloudcmd/create.go | 4 + cli/internal/cloudcmd/create_test.go | 10 ++ cli/internal/cloudcmd/rollback.go | 1 + cli/internal/cloudcmd/terminate.go | 3 + cli/internal/gcp/client/api.go | 28 +++ cli/internal/gcp/client/api_test.go | 137 +++++++++++++++ cli/internal/gcp/client/client.go | 74 +++++++- cli/internal/gcp/client/client_test.go | 156 +++++++++++++++++ cli/internal/gcp/client/gcpwrappers.go | 72 ++++++++ cli/internal/gcp/client/network.go | 135 +++++++++++++++ cli/internal/gcp/client/network_test.go | 162 ++++++++++++++++++ coordinator/cloudprovider/gcp/api.go | 9 + coordinator/cloudprovider/gcp/client.go | 57 +++++- coordinator/cloudprovider/gcp/client_test.go | 130 +++++++++++++- coordinator/cloudprovider/gcp/metadata.go | 14 +- .../cloudprovider/gcp/metadata_test.go | 6 + coordinator/cloudprovider/gcp/wrappers.go | 14 ++ internal/state/state.go | 3 + 21 files changed, 1043 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10e482e45..930eca021 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Early boot logging for Cloud Provider: GCP & Azure - Added `constellation-access-manager`, allowing users to manage SSH users over a ConfigMap. This allows persistent & dynamic management of SSH users on multiple nodes, even after a reboot. +- GCP-native Kubernetes load balancing ### Changed diff --git a/cli/internal/cloudcmd/clients.go b/cli/internal/cloudcmd/clients.go index 7cca22937..f93043e1e 100644 --- a/cli/internal/cloudcmd/clients.go +++ b/cli/internal/cloudcmd/clients.go @@ -14,9 +14,11 @@ type gcpclient interface { CreateVPCs(ctx context.Context) error CreateFirewall(ctx context.Context, input gcpcl.FirewallInput) error CreateInstances(ctx context.Context, input gcpcl.CreateInstancesInput) error + CreateLoadBalancer(ctx context.Context) error CreateServiceAccount(ctx context.Context, input gcpcl.ServiceAccountInput) (string, error) TerminateFirewall(ctx context.Context) error TerminateVPCs(context.Context) error + TerminateLoadBalancer(context.Context) error TerminateInstances(context.Context) error TerminateServiceAccount(ctx context.Context) error Close() error diff --git a/cli/internal/cloudcmd/clients_test.go b/cli/internal/cloudcmd/clients_test.go index aa52cccd2..b7119cf34 100644 --- a/cli/internal/cloudcmd/clients_test.go +++ b/cli/internal/cloudcmd/clients_test.go @@ -241,6 +241,11 @@ type fakeGcpClient struct { name string zone string serviceAccount string + + // loadbalancer + healthCheck string + backendService string + forwardingRule string } func (c *fakeGcpClient) GetState() (state.ConstellationState, error) { @@ -255,6 +260,9 @@ func (c *fakeGcpClient) GetState() (state.ConstellationState, error) { GCPNetwork: c.network, GCPSubnetwork: c.subnetwork, GCPFirewalls: c.firewalls, + GCPBackendService: c.backendService, + GCPHealthCheck: c.healthCheck, + GCPForwardingRule: c.forwardingRule, GCPProject: c.project, Name: c.name, UID: c.uid, @@ -279,6 +287,9 @@ func (c *fakeGcpClient) SetState(stat state.ConstellationState) error { c.uid = stat.UID c.zone = stat.GCPZone c.serviceAccount = stat.GCPServiceAccount + c.healthCheck = stat.GCPHealthCheck + c.backendService = stat.GCPBackendService + c.forwardingRule = stat.GCPForwardingRule return nil } @@ -332,6 +343,13 @@ func (c *fakeGcpClient) CreateServiceAccount(ctx context.Context, input gcpcl.Se }.ToCloudServiceAccountURI(), nil } +func (c *fakeGcpClient) CreateLoadBalancer(ctx context.Context) error { + c.healthCheck = "health-check" + c.backendService = "backend-service" + c.forwardingRule = "forwarding-rule" + return nil +} + func (c *fakeGcpClient) TerminateFirewall(ctx context.Context) error { if len(c.firewalls) == 0 { return nil @@ -364,6 +382,13 @@ func (c *fakeGcpClient) TerminateServiceAccount(context.Context) error { return nil } +func (c *fakeGcpClient) TerminateLoadBalancer(context.Context) error { + c.healthCheck = "" + c.backendService = "" + c.forwardingRule = "" + return nil +} + func (c *fakeGcpClient) Close() error { return nil } @@ -381,10 +406,12 @@ type stubGcpClient struct { createFirewallErr error createInstancesErr error createServiceAccountErr error + createLoadBalancerErr error terminateFirewallErr error terminateVPCsErr error terminateInstancesErr error terminateServiceAccountErr error + terminateLoadBalancerErr error closeErr error } @@ -412,6 +439,10 @@ func (c *stubGcpClient) CreateServiceAccount(ctx context.Context, input gcpcl.Se return gcpshared.ServiceAccountKey{}.ToCloudServiceAccountURI(), c.createServiceAccountErr } +func (c *stubGcpClient) CreateLoadBalancer(ctx context.Context) error { + return c.createLoadBalancerErr +} + func (c *stubGcpClient) TerminateFirewall(ctx context.Context) error { c.terminateFirewallCalled = true return c.terminateFirewallErr @@ -432,6 +463,10 @@ func (c *stubGcpClient) TerminateServiceAccount(context.Context) error { return c.terminateServiceAccountErr } +func (c *stubGcpClient) TerminateLoadBalancer(context.Context) error { + return c.terminateLoadBalancerErr +} + func (c *stubGcpClient) Close() error { c.closeCalled = true return c.closeErr diff --git a/cli/internal/cloudcmd/create.go b/cli/internal/cloudcmd/create.go index 2054ee07c..96af4d645 100644 --- a/cli/internal/cloudcmd/create.go +++ b/cli/internal/cloudcmd/create.go @@ -132,6 +132,10 @@ func (c *Creator) createGCP(ctx context.Context, cl gcpclient, config *config.Co return state.ConstellationState{}, err } + if err := cl.CreateLoadBalancer(ctx); err != nil { + return state.ConstellationState{}, err + } + return cl.GetState() } diff --git a/cli/internal/cloudcmd/create_test.go b/cli/internal/cloudcmd/create_test.go index 871d78009..b32bbde07 100644 --- a/cli/internal/cloudcmd/create_test.go +++ b/cli/internal/cloudcmd/create_test.go @@ -32,6 +32,9 @@ func TestCreator(t *testing.T) { GCPCoordinatorInstanceTemplate: "coordinator-template", GCPNetwork: "network", GCPSubnetwork: "subnetwork", + GCPBackendService: "backend-service", + GCPHealthCheck: "health-check", + GCPForwardingRule: "forwarding-rule", GCPFirewalls: []string{ "coordinator", "wireguard", "ssh", "nodeport", "kubernetes", "allow-cluster-internal-tcp", "allow-cluster-internal-udp", "allow-cluster-internal-icmp", @@ -103,6 +106,13 @@ func TestCreator(t *testing.T) { wantErr: true, wantRollback: true, }, + "gcp CreateLoadBalancer error": { + gcpclient: &stubGcpClient{createLoadBalancerErr: someErr}, + provider: cloudprovider.GCP, + config: config.Default(), + wantErr: true, + wantRollback: true, + }, "azure": { azureclient: &fakeAzureClient{}, provider: cloudprovider.Azure, diff --git a/cli/internal/cloudcmd/rollback.go b/cli/internal/cloudcmd/rollback.go index 5edc9725e..03489da6b 100644 --- a/cli/internal/cloudcmd/rollback.go +++ b/cli/internal/cloudcmd/rollback.go @@ -34,6 +34,7 @@ type rollbackerGCP struct { func (r *rollbackerGCP) rollback(ctx context.Context) error { var err error + err = multierr.Append(err, r.client.TerminateLoadBalancer(ctx)) err = multierr.Append(err, r.client.TerminateInstances(ctx)) err = multierr.Append(err, r.client.TerminateFirewall(ctx)) err = multierr.Append(err, r.client.TerminateVPCs(ctx)) diff --git a/cli/internal/cloudcmd/terminate.go b/cli/internal/cloudcmd/terminate.go index dc1adcc98..c3580e6d2 100644 --- a/cli/internal/cloudcmd/terminate.go +++ b/cli/internal/cloudcmd/terminate.go @@ -55,6 +55,9 @@ func (t *Terminator) terminateGCP(ctx context.Context, cl gcpclient, state state return err } + if err := cl.TerminateLoadBalancer(ctx); err != nil { + return err + } if err := cl.TerminateInstances(ctx); err != nil { return err } diff --git a/cli/internal/gcp/client/api.go b/cli/internal/gcp/client/api.go index b06055598..0c9bc6133 100644 --- a/cli/internal/gcp/client/api.go +++ b/cli/internal/gcp/client/api.go @@ -41,6 +41,34 @@ type firewallsAPI interface { opts ...gax.CallOption) (Operation, error) } +type forwardingRulesAPI interface { + Close() error + Delete(ctx context.Context, req *computepb.DeleteForwardingRuleRequest, + opts ...gax.CallOption) (Operation, error) + Insert(ctx context.Context, req *computepb.InsertForwardingRuleRequest, + opts ...gax.CallOption) (Operation, error) + Get(ctx context.Context, req *computepb.GetForwardingRuleRequest, + opts ...gax.CallOption) (*computepb.ForwardingRule, error) + SetLabels(ctx context.Context, req *computepb.SetLabelsForwardingRuleRequest, + opts ...gax.CallOption) (Operation, error) +} + +type backendServicesAPI interface { + Close() error + Delete(ctx context.Context, req *computepb.DeleteRegionBackendServiceRequest, + opts ...gax.CallOption) (Operation, error) + Insert(ctx context.Context, req *computepb.InsertRegionBackendServiceRequest, + opts ...gax.CallOption) (Operation, error) +} + +type healthChecksAPI interface { + Close() error + Delete(ctx context.Context, req *computepb.DeleteRegionHealthCheckRequest, + opts ...gax.CallOption) (Operation, error) + Insert(ctx context.Context, req *computepb.InsertRegionHealthCheckRequest, + opts ...gax.CallOption) (Operation, error) +} + type networksAPI interface { Close() error Delete(ctx context.Context, req *computepb.DeleteNetworkRequest, diff --git a/cli/internal/gcp/client/api_test.go b/cli/internal/gcp/client/api_test.go index 718bedf97..e5aebdbe9 100644 --- a/cli/internal/gcp/client/api_test.go +++ b/cli/internal/gcp/client/api_test.go @@ -219,6 +219,143 @@ func (a stubSubnetworksAPI) Delete(ctx context.Context, req *computepb.DeleteSub }, nil } +type stubBackendServicesAPI struct { + insertErr error + deleteErr error +} + +func (a stubBackendServicesAPI) Close() error { + return nil +} + +func (a stubBackendServicesAPI) Insert(ctx context.Context, req *computepb.InsertRegionBackendServiceRequest, + opts ...gax.CallOption, +) (Operation, error) { + if a.insertErr != nil { + return nil, a.insertErr + } + return &stubOperation{ + &computepb.Operation{ + Name: proto.String("name"), + Region: proto.String("region"), + }, + }, nil +} + +func (a stubBackendServicesAPI) Delete(ctx context.Context, req *computepb.DeleteRegionBackendServiceRequest, + opts ...gax.CallOption, +) (Operation, error) { + if a.deleteErr != nil { + return nil, a.deleteErr + } + return &stubOperation{ + &computepb.Operation{ + Name: proto.String("name"), + Region: proto.String("region"), + }, + }, nil +} + +type stubForwardingRulesAPI struct { + insertErr error + deleteErr error + getErr error + setLabelErr error + forwardingRule *computepb.ForwardingRule +} + +func (a stubForwardingRulesAPI) Close() error { + return nil +} + +func (a stubForwardingRulesAPI) Insert(ctx context.Context, req *computepb.InsertForwardingRuleRequest, + opts ...gax.CallOption, +) (Operation, error) { + if a.insertErr != nil { + return nil, a.insertErr + } + return &stubOperation{ + &computepb.Operation{ + Name: proto.String("name"), + Region: proto.String("region"), + }, + }, nil +} + +func (a stubForwardingRulesAPI) Delete(ctx context.Context, req *computepb.DeleteForwardingRuleRequest, + opts ...gax.CallOption, +) (Operation, error) { + if a.deleteErr != nil { + return nil, a.deleteErr + } + return &stubOperation{ + &computepb.Operation{ + Name: proto.String("name"), + Region: proto.String("region"), + }, + }, nil +} + +func (a stubForwardingRulesAPI) Get(ctx context.Context, req *computepb.GetForwardingRuleRequest, + opts ...gax.CallOption, +) (*computepb.ForwardingRule, error) { + if a.getErr != nil { + return nil, a.getErr + } + return a.forwardingRule, nil +} + +func (a stubForwardingRulesAPI) SetLabels(ctx context.Context, req *computepb.SetLabelsForwardingRuleRequest, + opts ...gax.CallOption, +) (Operation, error) { + if a.deleteErr != nil { + return nil, a.setLabelErr + } + return &stubOperation{ + &computepb.Operation{ + Name: proto.String("name"), + Region: proto.String("region"), + }, + }, nil +} + +type stubHealthChecksAPI struct { + insertErr error + deleteErr error +} + +func (a stubHealthChecksAPI) Close() error { + return nil +} + +func (a stubHealthChecksAPI) Insert(ctx context.Context, req *computepb.InsertRegionHealthCheckRequest, + opts ...gax.CallOption, +) (Operation, error) { + if a.insertErr != nil { + return nil, a.insertErr + } + return &stubOperation{ + &computepb.Operation{ + Name: proto.String("name"), + Region: proto.String("region"), + }, + }, nil +} + +func (a stubHealthChecksAPI) Delete(ctx context.Context, req *computepb.DeleteRegionHealthCheckRequest, + opts ...gax.CallOption, +) (Operation, error) { + if a.deleteErr != nil { + return nil, a.deleteErr + } + return &stubOperation{ + &computepb.Operation{ + Name: proto.String("name"), + Region: proto.String("region"), + }, + }, nil +} + type stubInstanceTemplateAPI struct { deleteErr error insertErr error diff --git a/cli/internal/gcp/client/client.go b/cli/internal/gcp/client/client.go index c5cf744ea..62ce50306 100644 --- a/cli/internal/gcp/client/client.go +++ b/cli/internal/gcp/client/client.go @@ -25,6 +25,9 @@ type Client struct { networksAPI subnetworksAPI firewallsAPI + forwardingRulesAPI + backendServicesAPI + healthChecksAPI instanceTemplateAPI instanceGroupManagersAPI iamAPI @@ -47,6 +50,11 @@ type Client struct { zone string region string serviceAccount string + + // loadbalancer + healthCheck string + backendService string + forwardingRule string } // NewFromDefault creates an uninitialized client. @@ -92,7 +100,31 @@ func NewFromDefault(ctx context.Context) (*Client, error) { _ = closeAll(closers) return nil, err } - closers = append(closers, fwAPI) + closers = append(closers, subnetAPI) + forwardingRulesAPI, err := compute.NewForwardingRulesRESTClient(ctx) + if err != nil { + _ = closeAll(closers) + return nil, err + } + closers = append(closers, forwardingRulesAPI) + backendServicesAPI, err := compute.NewRegionBackendServicesRESTClient(ctx) + if err != nil { + _ = closeAll(closers) + return nil, err + } + closers = append(closers, backendServicesAPI) + targetPoolsAPI, err := compute.NewTargetPoolsRESTClient(ctx) + if err != nil { + _ = closeAll(closers) + return nil, err + } + closers = append(closers, targetPoolsAPI) + healthChecksAPI, err := compute.NewRegionHealthChecksRESTClient(ctx) + if err != nil { + _ = closeAll(closers) + return nil, err + } + closers = append(closers, healthChecksAPI) templAPI, err := compute.NewInstanceTemplatesRESTClient(ctx) if err != nil { _ = closeAll(closers) @@ -124,6 +156,9 @@ func NewFromDefault(ctx context.Context) (*Client, error) { networksAPI: &networksClient{netAPI}, subnetworksAPI: &subnetworksClient{subnetAPI}, firewallsAPI: &firewallsClient{fwAPI}, + forwardingRulesAPI: &forwardingRulesClient{forwardingRulesAPI}, + backendServicesAPI: &backendServicesClient{backendServicesAPI}, + healthChecksAPI: &healthChecksClient{healthChecksAPI}, instanceTemplateAPI: &instanceTemplateClient{templAPI}, instanceGroupManagersAPI: &instanceGroupManagersClient{groupAPI}, iamAPI: &iamClient{iamAPI}, @@ -147,12 +182,19 @@ func NewInitialized(ctx context.Context, project, zone, region, name string) (*C func (c *Client) Close() error { closers := []closer{ c.instanceAPI, + c.operationRegionAPI, c.operationZoneAPI, c.operationGlobalAPI, c.networksAPI, + c.subnetworksAPI, c.firewallsAPI, + c.forwardingRulesAPI, + c.backendServicesAPI, + c.healthChecksAPI, c.instanceTemplateAPI, c.instanceGroupManagersAPI, + c.iamAPI, + c.projectsAPI, } return closeAll(closers) } @@ -246,6 +288,21 @@ func (c *Client) GetState() (state.ConstellationState, error) { } stat.GCPCoordinatorInstanceTemplate = c.coordinatorTemplate + if c.healthCheck == "" { + return state.ConstellationState{}, errors.New("client has no health check") + } + stat.GCPHealthCheck = c.healthCheck + + if c.backendService == "" { + return state.ConstellationState{}, errors.New("client has no backend service") + } + stat.GCPBackendService = c.backendService + + if c.forwardingRule == "" { + return state.ConstellationState{}, errors.New("client has no forwarding rule") + } + stat.GCPForwardingRule = c.forwardingRule + // service account does not have to be set at all times stat.GCPServiceAccount = c.serviceAccount @@ -327,6 +384,21 @@ func (c *Client) SetState(stat state.ConstellationState) error { } c.coordinatorTemplate = stat.GCPCoordinatorInstanceTemplate + if stat.GCPHealthCheck == "" { + return errors.New("state has no health check") + } + c.healthCheck = stat.GCPHealthCheck + + if stat.GCPBackendService == "" { + return errors.New("state has no backend service") + } + c.backendService = stat.GCPBackendService + + if stat.GCPForwardingRule == "" { + return errors.New("state has no forwarding rule") + } + c.forwardingRule = stat.GCPForwardingRule + // service account does not have to be set at all times c.serviceAccount = stat.GCPServiceAccount diff --git a/cli/internal/gcp/client/client_test.go b/cli/internal/gcp/client/client_test.go index d5669a9bd..2bf808296 100644 --- a/cli/internal/gcp/client/client_test.go +++ b/cli/internal/gcp/client/client_test.go @@ -44,6 +44,9 @@ func TestSetGetState(t *testing.T) { GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", GCPServiceAccount: "service-account", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, }, "missing nodes": { @@ -67,6 +70,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -91,6 +97,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -120,6 +129,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -149,6 +161,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -178,6 +193,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -207,6 +225,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -236,6 +257,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -264,6 +288,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -293,6 +320,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -322,6 +352,9 @@ func TestSetGetState(t *testing.T) { GCPSubnetwork: "subnet-id", GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -350,6 +383,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -379,6 +415,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -408,6 +447,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -437,6 +479,9 @@ func TestSetGetState(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -466,6 +511,9 @@ func TestSetGetState(t *testing.T) { GCPSubnetwork: "subnet-id", GCPFirewalls: []string{"fw-1", "fw-2"}, GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", }, wantErr: true, }, @@ -495,6 +543,105 @@ func TestSetGetState(t *testing.T) { GCPSubnetwork: "subnet-id", GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", + }, + wantErr: true, + }, + "missing backend service": { + state: state.ConstellationState{ + CloudProvider: cloudprovider.GCP.String(), + GCPNodes: cloudtypes.Instances{ + "id-1": { + PublicIP: "ip1", + PrivateIP: "ip2", + }, + }, + GCPCoordinators: cloudtypes.Instances{ + "id-1": { + PublicIP: "ip3", + PrivateIP: "ip4", + }, + }, + GCPNodeInstanceGroup: "group-id", + GCPCoordinatorInstanceGroup: "group-id", + GCPProject: "proj-id", + GCPZone: "zone-id", + GCPRegion: "region-id", + Name: "name", + UID: "uid", + GCPNetwork: "net-id", + GCPSubnetwork: "subnet-id", + GCPFirewalls: []string{"fw-1", "fw-2"}, + GCPNodeInstanceTemplate: "temp-id", + GCPCoordinatorInstanceTemplate: "temp-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", + }, + wantErr: true, + }, + "missing health check": { + state: state.ConstellationState{ + CloudProvider: cloudprovider.GCP.String(), + GCPNodes: cloudtypes.Instances{ + "id-1": { + PublicIP: "ip1", + PrivateIP: "ip2", + }, + }, + GCPCoordinators: cloudtypes.Instances{ + "id-1": { + PublicIP: "ip3", + PrivateIP: "ip4", + }, + }, + GCPNodeInstanceGroup: "group-id", + GCPCoordinatorInstanceGroup: "group-id", + GCPProject: "proj-id", + GCPZone: "zone-id", + GCPRegion: "region-id", + Name: "name", + UID: "uid", + GCPNetwork: "net-id", + GCPSubnetwork: "subnet-id", + GCPFirewalls: []string{"fw-1", "fw-2"}, + GCPNodeInstanceTemplate: "temp-id", + GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPForwardingRule: "forwarding-rule-id", + }, + wantErr: true, + }, + "missing forwarding rule": { + state: state.ConstellationState{ + CloudProvider: cloudprovider.GCP.String(), + GCPNodes: cloudtypes.Instances{ + "id-1": { + PublicIP: "ip1", + PrivateIP: "ip2", + }, + }, + GCPCoordinators: cloudtypes.Instances{ + "id-1": { + PublicIP: "ip3", + PrivateIP: "ip4", + }, + }, + GCPNodeInstanceGroup: "group-id", + GCPCoordinatorInstanceGroup: "group-id", + GCPProject: "proj-id", + GCPZone: "zone-id", + GCPRegion: "region-id", + Name: "name", + UID: "uid", + GCPNetwork: "net-id", + GCPSubnetwork: "subnet-id", + GCPFirewalls: []string{"fw-1", "fw-2"}, + GCPNodeInstanceTemplate: "temp-id", + GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", }, wantErr: true, }, @@ -549,6 +696,9 @@ func TestSetGetState(t *testing.T) { nodeTemplate: tc.state.GCPNodeInstanceTemplate, coordinatorTemplate: tc.state.GCPCoordinatorInstanceTemplate, serviceAccount: tc.state.GCPServiceAccount, + healthCheck: tc.state.GCPHealthCheck, + backendService: tc.state.GCPBackendService, + forwardingRule: tc.state.GCPForwardingRule, } if tc.wantErr { _, err := client.GetState() @@ -592,6 +742,9 @@ func TestSetStateCloudProvider(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", } assert.Error(client.SetState(stateMissingCloudProvider)) stateIncorrectCloudProvider := state.ConstellationState{ @@ -620,6 +773,9 @@ func TestSetStateCloudProvider(t *testing.T) { GCPFirewalls: []string{"fw-1", "fw-2"}, GCPNodeInstanceTemplate: "temp-id", GCPCoordinatorInstanceTemplate: "temp-id", + GCPBackendService: "backend-service-id", + GCPHealthCheck: "health-check-id", + GCPForwardingRule: "forwarding-rule-id", } assert.Error(client.SetState(stateIncorrectCloudProvider)) } diff --git a/cli/internal/gcp/client/gcpwrappers.go b/cli/internal/gcp/client/gcpwrappers.go index aca37fd55..c9337f9fa 100644 --- a/cli/internal/gcp/client/gcpwrappers.go +++ b/cli/internal/gcp/client/gcpwrappers.go @@ -46,6 +46,78 @@ func (c *firewallsClient) Insert(ctx context.Context, req *computepb.InsertFirew return c.FirewallsClient.Insert(ctx, req) } +type forwardingRulesClient struct { + *compute.ForwardingRulesClient +} + +func (c *forwardingRulesClient) Close() error { + return c.ForwardingRulesClient.Close() +} + +func (c *forwardingRulesClient) Delete(ctx context.Context, req *computepb.DeleteForwardingRuleRequest, + opts ...gax.CallOption, +) (Operation, error) { + return c.ForwardingRulesClient.Delete(ctx, req) +} + +func (c *forwardingRulesClient) Insert(ctx context.Context, req *computepb.InsertForwardingRuleRequest, + opts ...gax.CallOption, +) (Operation, error) { + return c.ForwardingRulesClient.Insert(ctx, req) +} + +func (c *forwardingRulesClient) Get(ctx context.Context, req *computepb.GetForwardingRuleRequest, + opts ...gax.CallOption, +) (*computepb.ForwardingRule, error) { + return c.ForwardingRulesClient.Get(ctx, req) +} + +func (c *forwardingRulesClient) SetLabels(ctx context.Context, req *computepb.SetLabelsForwardingRuleRequest, + opts ...gax.CallOption, +) (Operation, error) { + return c.ForwardingRulesClient.SetLabels(ctx, req) +} + +type backendServicesClient struct { + *compute.RegionBackendServicesClient +} + +func (c *backendServicesClient) Close() error { + return c.RegionBackendServicesClient.Close() +} + +func (c *backendServicesClient) Insert(ctx context.Context, req *computepb.InsertRegionBackendServiceRequest, + opts ...gax.CallOption, +) (Operation, error) { + return c.RegionBackendServicesClient.Insert(ctx, req) +} + +func (c *backendServicesClient) Delete(ctx context.Context, req *computepb.DeleteRegionBackendServiceRequest, + opts ...gax.CallOption, +) (Operation, error) { + return c.RegionBackendServicesClient.Delete(ctx, req) +} + +type healthChecksClient struct { + *compute.RegionHealthChecksClient +} + +func (c *healthChecksClient) Close() error { + return c.RegionHealthChecksClient.Close() +} + +func (c *healthChecksClient) Delete(ctx context.Context, req *computepb.DeleteRegionHealthCheckRequest, + opts ...gax.CallOption, +) (Operation, error) { + return c.RegionHealthChecksClient.Delete(ctx, req) +} + +func (c *healthChecksClient) Insert(ctx context.Context, req *computepb.InsertRegionHealthCheckRequest, + opts ...gax.CallOption, +) (Operation, error) { + return c.RegionHealthChecksClient.Insert(ctx, req) +} + type networksClient struct { *compute.NetworksClient } diff --git a/cli/internal/gcp/client/network.go b/cli/internal/gcp/client/network.go index 326a8476e..a64e36a44 100644 --- a/cli/internal/gcp/client/network.go +++ b/cli/internal/gcp/client/network.go @@ -3,8 +3,10 @@ package client import ( "context" "errors" + "fmt" "github.com/edgelesssys/constellation/internal/cloud/cloudtypes" + "google.golang.org/genproto/googleapis/cloud/compute/v1" computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" "google.golang.org/protobuf/proto" ) @@ -197,3 +199,136 @@ func (c *Client) terminateSubnet(ctx context.Context) error { } return c.waitForOperations(ctx, []Operation{op}) } + +// CreateLoadBalancer creates a load balancer. +func (c *Client) CreateLoadBalancer(ctx context.Context) error { + c.healthCheck = c.name + "-" + c.uid + resp, err := c.healthChecksAPI.Insert(ctx, &computepb.InsertRegionHealthCheckRequest{ + Project: c.project, + Region: c.region, + HealthCheckResource: &computepb.HealthCheck{ + Name: proto.String(c.healthCheck), + Type: proto.String(compute.HealthCheck_Type_name[int32(compute.HealthCheck_TCP)]), + CheckIntervalSec: proto.Int32(1), + TimeoutSec: proto.Int32(1), + TcpHealthCheck: &computepb.TCPHealthCheck{ + Port: proto.Int32(6443), + }, + }, + }) + if err != nil { + return err + } + if err := c.waitForOperations(ctx, []Operation{resp}); err != nil { + return err + } + + c.backendService = c.name + "-" + c.uid + resp, err = c.backendServicesAPI.Insert(ctx, &computepb.InsertRegionBackendServiceRequest{ + Project: c.project, + Region: c.region, + BackendServiceResource: &computepb.BackendService{ + Name: proto.String(c.backendService), + Protocol: proto.String(compute.BackendService_Protocol_name[int32(compute.BackendService_TCP)]), + LoadBalancingScheme: proto.String(computepb.BackendService_LoadBalancingScheme_name[int32(compute.BackendService_EXTERNAL)]), + TimeoutSec: proto.Int32(10), + HealthChecks: []string{"https://www.googleapis.com/compute/v1/projects/" + c.project + "/regions/" + c.region + "/healthChecks/" + c.healthCheck}, + Backends: []*computepb.Backend{ + { + BalancingMode: proto.String(computepb.Backend_BalancingMode_name[int32(compute.Backend_CONNECTION)]), + Group: proto.String("https://www.googleapis.com/compute/v1/projects/" + c.project + "/zones/" + c.zone + "/instanceGroups/" + c.coordinatorInstanceGroup), + }, + }, + }, + }) + if err != nil { + return err + } + if err := c.waitForOperations(ctx, []Operation{resp}); err != nil { + return err + } + + c.forwardingRule = c.name + "-" + c.uid + resp, err = c.forwardingRulesAPI.Insert(ctx, &computepb.InsertForwardingRuleRequest{ + Project: c.project, + Region: c.region, + ForwardingRuleResource: &computepb.ForwardingRule{ + Name: proto.String(c.forwardingRule), + IPProtocol: proto.String(compute.ForwardingRule_IPProtocolEnum_name[int32(compute.ForwardingRule_TCP)]), + LoadBalancingScheme: proto.String(compute.ForwardingRule_LoadBalancingScheme_name[int32(compute.ForwardingRule_EXTERNAL)]), + Ports: []string{"6443", "9000"}, + BackendService: proto.String("https://www.googleapis.com/compute/v1/projects/" + c.project + "/regions/" + c.region + "/backendServices/" + c.backendService), + }, + }) + if err != nil { + return err + } + if err := c.waitForOperations(ctx, []Operation{resp}); err != nil { + return err + } + + forwardingRule, err := c.forwardingRulesAPI.Get(ctx, &computepb.GetForwardingRuleRequest{ + Project: c.project, + Region: c.region, + ForwardingRule: c.forwardingRule, + }) + if err != nil { + return err + } + if forwardingRule.LabelFingerprint == nil { + return fmt.Errorf("forwarding rule %s has no label fingerprint", c.forwardingRule) + } + + resp, err = c.forwardingRulesAPI.SetLabels(ctx, &computepb.SetLabelsForwardingRuleRequest{ + Project: c.project, + Region: c.region, + Resource: c.forwardingRule, + RegionSetLabelsRequestResource: &computepb.RegionSetLabelsRequest{ + Labels: map[string]string{"constellation-uid": c.uid}, + LabelFingerprint: forwardingRule.LabelFingerprint, + }, + }) + if err != nil { + return err + } + + return c.waitForOperations(ctx, []Operation{resp}) +} + +// TerminteLoadBalancer removes the load balancer and its associated resources. +func (c *Client) TerminateLoadBalancer(ctx context.Context) error { + resp, err := c.forwardingRulesAPI.Delete(ctx, &computepb.DeleteForwardingRuleRequest{ + Project: c.project, + Region: c.region, + ForwardingRule: c.forwardingRule, + }) + if err != nil { + return err + } + if err := c.waitForOperations(ctx, []Operation{resp}); err != nil { + return err + } + + resp, err = c.backendServicesAPI.Delete(ctx, &computepb.DeleteRegionBackendServiceRequest{ + Project: c.project, + Region: c.region, + BackendService: c.backendService, + }) + if err != nil { + return err + } + if err := c.waitForOperations(ctx, []Operation{resp}); err != nil { + return err + } + + resp, err = c.healthChecksAPI.Delete(ctx, &computepb.DeleteRegionHealthCheckRequest{ + Project: c.project, + Region: c.region, + HealthCheck: c.healthCheck, + }) + if err != nil { + return err + } + + return c.waitForOperations(ctx, []Operation{resp}) +} diff --git a/cli/internal/gcp/client/network_test.go b/cli/internal/gcp/client/network_test.go index f7270b65c..3e2979926 100644 --- a/cli/internal/gcp/client/network_test.go +++ b/cli/internal/gcp/client/network_test.go @@ -7,6 +7,8 @@ import ( "github.com/edgelesssys/constellation/internal/cloud/cloudtypes" "github.com/stretchr/testify/assert" + "google.golang.org/genproto/googleapis/cloud/compute/v1" + "google.golang.org/protobuf/proto" ) func TestCreateVPCs(t *testing.T) { @@ -307,3 +309,163 @@ func TestTerminateFirewall(t *testing.T) { }) } } + +func TestCreateLoadBalancer(t *testing.T) { + someErr := errors.New("failed") + testCases := map[string]struct { + operationRegionAPI operationRegionAPI + healthChecksAPI healthChecksAPI + backendServicesAPI backendServicesAPI + forwardingRulesAPI forwardingRulesAPI + wantErr bool + }{ + "successful create": { + healthChecksAPI: stubHealthChecksAPI{}, + backendServicesAPI: stubBackendServicesAPI{}, + forwardingRulesAPI: stubForwardingRulesAPI{forwardingRule: &compute.ForwardingRule{LabelFingerprint: proto.String("fingerprint")}}, + operationRegionAPI: stubOperationRegionAPI{}, + }, + "CreateLoadBalancer fails when getting forwarding rule": { + healthChecksAPI: stubHealthChecksAPI{}, + backendServicesAPI: stubBackendServicesAPI{}, + forwardingRulesAPI: stubForwardingRulesAPI{getErr: someErr}, + operationRegionAPI: stubOperationRegionAPI{}, + wantErr: true, + }, + "CreateLoadBalancer fails when label fingerprint is missing": { + healthChecksAPI: stubHealthChecksAPI{}, + backendServicesAPI: stubBackendServicesAPI{}, + forwardingRulesAPI: stubForwardingRulesAPI{forwardingRule: &compute.ForwardingRule{}}, + operationRegionAPI: stubOperationRegionAPI{}, + wantErr: true, + }, + "CreateLoadBalancer fails when creating health check": { + healthChecksAPI: stubHealthChecksAPI{insertErr: someErr}, + backendServicesAPI: stubBackendServicesAPI{}, + forwardingRulesAPI: stubForwardingRulesAPI{forwardingRule: &compute.ForwardingRule{LabelFingerprint: proto.String("fingerprint")}}, + operationRegionAPI: stubOperationRegionAPI{}, + wantErr: true, + }, + "CreateLoadBalancer fails when creating backend service": { + healthChecksAPI: stubHealthChecksAPI{}, + backendServicesAPI: stubBackendServicesAPI{insertErr: someErr}, + forwardingRulesAPI: stubForwardingRulesAPI{}, + operationRegionAPI: stubOperationRegionAPI{}, + wantErr: true, + }, + "CreateLoadBalancer fails when creating forwarding rule": { + healthChecksAPI: stubHealthChecksAPI{}, + backendServicesAPI: stubBackendServicesAPI{}, + forwardingRulesAPI: stubForwardingRulesAPI{insertErr: someErr}, + operationRegionAPI: stubOperationRegionAPI{}, + wantErr: true, + }, + "CreateLoadBalancer fails when waiting on operation": { + healthChecksAPI: stubHealthChecksAPI{}, + backendServicesAPI: stubBackendServicesAPI{}, + forwardingRulesAPI: stubForwardingRulesAPI{forwardingRule: &compute.ForwardingRule{LabelFingerprint: proto.String("fingerprint")}}, + operationRegionAPI: stubOperationRegionAPI{waitErr: someErr}, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + ctx := context.Background() + client := Client{ + project: "project", + zone: "zone", + name: "name", + uid: "uid", + backendServicesAPI: tc.backendServicesAPI, + forwardingRulesAPI: tc.forwardingRulesAPI, + healthChecksAPI: tc.healthChecksAPI, + operationRegionAPI: tc.operationRegionAPI, + } + + if tc.wantErr { + assert.Error(client.CreateLoadBalancer(ctx)) + } else { + assert.NoError(client.CreateLoadBalancer(ctx)) + assert.NotEmpty(client.healthCheck) + assert.NotEmpty(client.backendService) + assert.NotEmpty(client.forwardingRule) + } + }) + } +} + +func TestTerminateLoadBalancer(t *testing.T) { + someErr := errors.New("failed") + testCases := map[string]struct { + operationRegionAPI operationRegionAPI + healthChecksAPI healthChecksAPI + backendServicesAPI backendServicesAPI + forwardingRulesAPI forwardingRulesAPI + wantErr bool + }{ + "successful terminate": { + healthChecksAPI: stubHealthChecksAPI{}, + backendServicesAPI: stubBackendServicesAPI{}, + forwardingRulesAPI: stubForwardingRulesAPI{}, + operationRegionAPI: stubOperationRegionAPI{}, + }, + "TerminateLoadBalancer fails when deleting health check": { + healthChecksAPI: stubHealthChecksAPI{deleteErr: someErr}, + backendServicesAPI: stubBackendServicesAPI{}, + forwardingRulesAPI: stubForwardingRulesAPI{}, + operationRegionAPI: stubOperationRegionAPI{}, + wantErr: true, + }, + "TerminateLoadBalancer fails when deleting backend service": { + healthChecksAPI: stubHealthChecksAPI{}, + backendServicesAPI: stubBackendServicesAPI{deleteErr: someErr}, + forwardingRulesAPI: stubForwardingRulesAPI{}, + operationRegionAPI: stubOperationRegionAPI{}, + wantErr: true, + }, + "TerminateLoadBalancer fails when deleting forwarding rule": { + healthChecksAPI: stubHealthChecksAPI{}, + backendServicesAPI: stubBackendServicesAPI{}, + forwardingRulesAPI: stubForwardingRulesAPI{deleteErr: someErr}, + operationRegionAPI: stubOperationRegionAPI{}, + wantErr: true, + }, + "TerminateLoadBalancer fails when waiting on operation": { + healthChecksAPI: stubHealthChecksAPI{}, + backendServicesAPI: stubBackendServicesAPI{}, + forwardingRulesAPI: stubForwardingRulesAPI{}, + operationRegionAPI: stubOperationRegionAPI{waitErr: someErr}, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + ctx := context.Background() + client := Client{ + project: "project", + zone: "zone", + name: "name", + uid: "uid", + backendServicesAPI: tc.backendServicesAPI, + forwardingRulesAPI: tc.forwardingRulesAPI, + healthChecksAPI: tc.healthChecksAPI, + operationRegionAPI: tc.operationRegionAPI, + } + + if tc.wantErr { + assert.Error(client.TerminateLoadBalancer(ctx)) + } else { + assert.NoError(client.TerminateLoadBalancer(ctx)) + assert.Empty(client.healthCheck) + assert.Empty(client.backendService) + assert.Empty(client.forwardingRule) + } + }) + } +} diff --git a/coordinator/cloudprovider/gcp/api.go b/coordinator/cloudprovider/gcp/api.go index 033b08446..05078a30a 100644 --- a/coordinator/cloudprovider/gcp/api.go +++ b/coordinator/cloudprovider/gcp/api.go @@ -22,6 +22,11 @@ type subnetworkAPI interface { Close() error } +type forwardingRulesAPI interface { + List(ctx context.Context, req *computepb.ListForwardingRulesRequest, opts ...gax.CallOption) ForwardingRuleIterator + Close() error +} + type metadataAPI interface { InstanceAttributeValue(attr string) (string, error) ProjectID() (string, error) @@ -40,3 +45,7 @@ type InstanceIterator interface { type SubnetworkIterator interface { Next() (*computepb.Subnetwork, error) } + +type ForwardingRuleIterator interface { + Next() (*computepb.ForwardingRule, error) +} diff --git a/coordinator/cloudprovider/gcp/client.go b/coordinator/cloudprovider/gcp/client.go index f3ebeb84f..3a2600db7 100644 --- a/coordinator/cloudprovider/gcp/client.go +++ b/coordinator/cloudprovider/gcp/client.go @@ -3,6 +3,7 @@ package gcp import ( "context" "fmt" + "regexp" "strings" compute "cloud.google.com/go/compute/apiv1" @@ -17,11 +18,14 @@ import ( const gcpSSHMetadataKey = "ssh-keys" +var zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])") + // Client implements the gcp.API interface. type Client struct { instanceAPI subnetworkAPI metadataAPI + forwardingRulesAPI } // NewClient creates a new Client. @@ -34,7 +38,16 @@ func NewClient(ctx context.Context) (*Client, error) { if err != nil { return nil, err } - return &Client{instanceAPI: &instanceClient{insAPI}, subnetworkAPI: &subnetworkClient{subnetAPI}, metadataAPI: &metadataClient{}}, nil + forwardingRulesAPI, err := compute.NewForwardingRulesRESTClient(ctx) + if err != nil { + return nil, err + } + return &Client{ + instanceAPI: &instanceClient{insAPI}, + subnetworkAPI: &subnetworkClient{subnetAPI}, + forwardingRulesAPI: &forwardingRulesClient{forwardingRulesAPI}, + metadataAPI: &metadataClient{}, + }, nil } // RetrieveInstances returns list of instances including their ips and metadata. @@ -178,8 +191,10 @@ func (c *Client) RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, // convert: // zone --> region // europe-west3-b --> europe-west3 - regionParts := strings.Split(zone, "-") - region := strings.TrimSuffix(zone, "-"+regionParts[len(regionParts)-1]) + region := zoneFromRegionRegex.FindString(zone) + if region == "" { + return "", fmt.Errorf("invalid zone %s", zone) + } req := &computepb.GetSubnetworkRequest{ Project: project, @@ -196,11 +211,47 @@ func (c *Client) RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, return *subnetwork.IpCidrRange, nil } +// RetrieveLoadBalancerIP returns the IP address of the load balancer specified by project, zone and loadBalancerName. +func (c *Client) RetrieveLoadBalancerIP(ctx context.Context, project, zone string) (string, error) { + uid, err := c.uid() + if err != nil { + return "", err + } + + region := zoneFromRegionRegex.FindString(zone) + if region == "" { + return "", fmt.Errorf("invalid zone %s", zone) + } + + req := &computepb.ListForwardingRulesRequest{ + Region: region, + Project: project, + } + iter := c.forwardingRulesAPI.List(ctx, req) + for { + resp, err := iter.Next() + if err == iterator.Done { + break + } + if err != nil { + return "", fmt.Errorf("retrieving load balancer IP failed: %w", err) + } + if resp.Labels["constellation-uid"] == uid { + return *resp.IPAddress, nil + } + } + + return "", fmt.Errorf("retrieving load balancer IP failed: load balancer not found") +} + // Close closes the instanceAPI client. func (c *Client) Close() error { if err := c.subnetworkAPI.Close(); err != nil { return err } + if err := c.forwardingRulesAPI.Close(); err != nil { + return err + } return c.instanceAPI.Close() } diff --git a/coordinator/cloudprovider/gcp/client_test.go b/coordinator/cloudprovider/gcp/client_test.go index dc965348a..c7680e8b9 100644 --- a/coordinator/cloudprovider/gcp/client_test.go +++ b/coordinator/cloudprovider/gcp/client_test.go @@ -756,7 +756,7 @@ func TestRetrieveSubnetworkAliasCIDR(t *testing.T) { require := require.New(t) client := Client{instanceAPI: tc.stubInstancesClient, subnetworkAPI: tc.stubSubnetworksClient} - aliasCIDR, err := client.RetrieveSubnetworkAliasCIDR(context.Background(), "project", "zone", "subnetwork") + aliasCIDR, err := client.RetrieveSubnetworkAliasCIDR(context.Background(), "project", "us-central1-a", "subnetwork") if tc.wantErr { assert.Error(err) @@ -768,18 +768,106 @@ func TestRetrieveSubnetworkAliasCIDR(t *testing.T) { } } +func TestRetrieveLoadBalancerIP(t *testing.T) { + loadBalancerIP := "192.0.2.1" + uid := "uid" + someErr := errors.New("some error") + testCases := map[string]struct { + stubForwardingRulesClient stubForwardingRulesClient + stubMetadataClient stubMetadataClient + wantLoadBalancerIP string + wantErr bool + }{ + "RetrieveSubnetworkAliasCIDR works": { + stubMetadataClient: stubMetadataClient{InstanceValue: uid}, + stubForwardingRulesClient: stubForwardingRulesClient{ + ForwardingRuleIterator: &stubForwardingRuleIterator{ + rules: []*computepb.ForwardingRule{ + { + IPAddress: proto.String(loadBalancerIP), + Labels: map[string]string{"constellation-uid": uid}, + }, + }, + }, + }, + wantLoadBalancerIP: loadBalancerIP, + }, + "RetrieveSubnetworkAliasCIDR fails when no matching load balancers exists": { + stubMetadataClient: stubMetadataClient{InstanceValue: uid}, + stubForwardingRulesClient: stubForwardingRulesClient{ + ForwardingRuleIterator: &stubForwardingRuleIterator{ + rules: []*computepb.ForwardingRule{ + { + IPAddress: proto.String(loadBalancerIP), + }, + }, + }, + }, + wantErr: true, + }, + "RetrieveSubnetworkAliasCIDR fails when retrieving uid": { + stubMetadataClient: stubMetadataClient{InstanceErr: someErr}, + stubForwardingRulesClient: stubForwardingRulesClient{ + ForwardingRuleIterator: &stubForwardingRuleIterator{ + rules: []*computepb.ForwardingRule{ + { + IPAddress: proto.String(loadBalancerIP), + Labels: map[string]string{"constellation-uid": uid}, + }, + }, + }, + }, + wantErr: true, + }, + "RetrieveSubnetworkAliasCIDR fails when retrieving loadbalancer IP": { + stubMetadataClient: stubMetadataClient{}, + stubForwardingRulesClient: stubForwardingRulesClient{ + ForwardingRuleIterator: &stubForwardingRuleIterator{ + nextErr: someErr, + rules: []*computepb.ForwardingRule{ + { + IPAddress: proto.String(loadBalancerIP), + Labels: map[string]string{"constellation-uid": uid}, + }, + }, + }, + }, + wantErr: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{forwardingRulesAPI: tc.stubForwardingRulesClient, metadataAPI: tc.stubMetadataClient} + aliasCIDR, err := client.RetrieveLoadBalancerIP(context.Background(), "project", "us-central1-a") + + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Equal(tc.wantLoadBalancerIP, aliasCIDR) + }) + } +} + func TestClose(t *testing.T) { someErr := errors.New("failed") assert := assert.New(t) - client := Client{instanceAPI: stubInstancesClient{}, subnetworkAPI: stubSubnetworksClient{}} + client := Client{instanceAPI: stubInstancesClient{}, subnetworkAPI: stubSubnetworksClient{}, forwardingRulesAPI: stubForwardingRulesClient{}} assert.NoError(client.Close()) - client = Client{instanceAPI: stubInstancesClient{CloseErr: someErr}, subnetworkAPI: stubSubnetworksClient{}} + client = Client{instanceAPI: stubInstancesClient{CloseErr: someErr}, subnetworkAPI: stubSubnetworksClient{}, forwardingRulesAPI: stubForwardingRulesClient{}} assert.Error(client.Close()) - client = Client{instanceAPI: stubInstancesClient{}, subnetworkAPI: stubSubnetworksClient{CloseErr: someErr}} + client = Client{instanceAPI: stubInstancesClient{}, subnetworkAPI: stubSubnetworksClient{CloseErr: someErr}, forwardingRulesAPI: stubForwardingRulesClient{}} + assert.Error(client.Close()) + + client = Client{instanceAPI: stubInstancesClient{}, subnetworkAPI: stubSubnetworksClient{}, forwardingRulesAPI: stubForwardingRulesClient{CloseErr: someErr}} assert.Error(client.Close()) } @@ -895,6 +983,40 @@ func (s stubSubnetworksClient) Close() error { return s.CloseErr } +type stubForwardingRuleIterator struct { + rules []*computepb.ForwardingRule + nextErr error + + internalCounter int +} + +func (i *stubForwardingRuleIterator) Next() (*computepb.ForwardingRule, error) { + if i.nextErr != nil { + return nil, i.nextErr + } + if i.internalCounter >= len(i.rules) { + i.internalCounter = 0 + return nil, iterator.Done + } + resp := i.rules[i.internalCounter] + i.internalCounter++ + return resp, nil +} + +type stubForwardingRulesClient struct { + ForwardingRuleIterator ForwardingRuleIterator + GetErr error + CloseErr error +} + +func (s stubForwardingRulesClient) List(ctx context.Context, req *computepb.ListForwardingRulesRequest, opts ...gax.CallOption) ForwardingRuleIterator { + return s.ForwardingRuleIterator +} + +func (s stubForwardingRulesClient) Close() error { + return s.CloseErr +} + type stubMetadataClient struct { InstanceValue string InstanceErr error diff --git a/coordinator/cloudprovider/gcp/metadata.go b/coordinator/cloudprovider/gcp/metadata.go index f3f114003..e2f54eb15 100644 --- a/coordinator/cloudprovider/gcp/metadata.go +++ b/coordinator/cloudprovider/gcp/metadata.go @@ -26,6 +26,8 @@ type API interface { RetrieveInstanceName() (string, error) // RetrieveSubnetworkAliasCIDR retrieves the subnetwork CIDR of the current instance. RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, instanceName string) (string, error) + // RetrieveLoadBalancerIP retrieves the load balancer IP of the current instance. + RetrieveLoadBalancerIP(ctx context.Context, project, zone string) (string, error) // SetInstanceMetadata sets metadata key: value of the instance specified by project, zone and instanceName. SetInstanceMetadata(ctx context.Context, project, zone, instanceName, key, value string) error // UnsetInstanceMetadata removes a metadata key-value pair of the instance specified by project, zone and instanceName. @@ -140,12 +142,20 @@ func (m *Metadata) GetSubnetworkCIDR(ctx context.Context) (string, error) { // SupportsLoadBalancer returns true if the cloud provider supports load balancers. func (m *Metadata) SupportsLoadBalancer() bool { - return false + return true } // GetLoadBalancerIP returns the IP of the load balancer. func (m *Metadata) GetLoadBalancerIP(ctx context.Context) (string, error) { - return "", nil + project, err := m.api.RetrieveProjectID() + if err != nil { + return "", err + } + zone, err := m.api.RetrieveZone() + if err != nil { + return "", err + } + return m.api.RetrieveLoadBalancerIP(ctx, project, zone) } // Supported is used to determine if metadata API is implemented for this cloud provider. diff --git a/coordinator/cloudprovider/gcp/metadata_test.go b/coordinator/cloudprovider/gcp/metadata_test.go index d46f7bf6c..1cc858e86 100644 --- a/coordinator/cloudprovider/gcp/metadata_test.go +++ b/coordinator/cloudprovider/gcp/metadata_test.go @@ -368,11 +368,13 @@ type stubGCPClient struct { projectID string zone string instanceName string + loadBalancerIP string retrieveProjectIDErr error retrieveZoneErr error retrieveInstanceNameErr error setInstanceMetadataErr error unsetInstanceMetadataErr error + retrieveLoadBalancerErr error instanceMetadataProjects []string instanceMetadataZones []string @@ -410,6 +412,10 @@ func (s *stubGCPClient) RetrieveInstanceName() (string, error) { return s.instanceName, s.retrieveInstanceNameErr } +func (s *stubGCPClient) RetrieveLoadBalancerIP(ctx context.Context, project, zone string) (string, error) { + return s.loadBalancerIP, s.retrieveLoadBalancerErr +} + func (s *stubGCPClient) SetInstanceMetadata(ctx context.Context, project, zone, instanceName, key, value string) error { s.instanceMetadataProjects = append(s.instanceMetadataProjects, project) s.instanceMetadataZones = append(s.instanceMetadataZones, zone) diff --git a/coordinator/cloudprovider/gcp/wrappers.go b/coordinator/cloudprovider/gcp/wrappers.go index 061a46ec0..c309c95e6 100644 --- a/coordinator/cloudprovider/gcp/wrappers.go +++ b/coordinator/cloudprovider/gcp/wrappers.go @@ -43,6 +43,20 @@ func (c *subnetworkClient) Get(ctx context.Context, req *computepb.GetSubnetwork return c.SubnetworksClient.Get(ctx, req) } +type forwardingRulesClient struct { + *compute.ForwardingRulesClient +} + +func (c *forwardingRulesClient) Close() error { + return c.ForwardingRulesClient.Close() +} + +func (c *forwardingRulesClient) List(ctx context.Context, req *computepb.ListForwardingRulesRequest, + opts ...gax.CallOption, +) ForwardingRuleIterator { + return c.ForwardingRulesClient.List(ctx, req) +} + type metadataClient struct{} func (c *metadataClient) InstanceAttributeValue(attr string) (string, error) { diff --git a/internal/state/state.go b/internal/state/state.go index 78a24ed72..410365762 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -19,6 +19,9 @@ type ConstellationState struct { GCPNetwork string `json:"gcpnetwork,omitempty"` GCPSubnetwork string `json:"gcpsubnetwork,omitempty"` GCPFirewalls []string `json:"gcpfirewalls,omitempty"` + GCPBackendService string `json:"gcpbackendservice,omitempty"` + GCPHealthCheck string `json:"gcphealthcheck,omitempty"` + GCPForwardingRule string `json:"gcpforwardingrule,omitempty"` GCPProject string `json:"gcpproject,omitempty"` GCPZone string `json:"gcpzone,omitempty"` GCPRegion string `json:"gcpregion,omitempty"`