Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-09-20 13:04:36 -04:00
Code Issues Projects Releases Packages Wiki Activity

gcp: unofficial support for internal LBs (#826)

Browse source
This commit is contained in:
3u13r 2022-12-28 13:30:39 +01:00 • committed by GitHub
parent 9859b30c4d
commit 47fb9f7cc8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 982 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

100
internal/cloud/gcp/cloud.go
View file

@ -31,14 +31,18 @@ const (
tagUsage = "constellation-use"
)
var zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])")
var (
zoneFromRegionRegex = regexp.MustCompile("([a-z]*-[a-z]*[0-9])")
errNoForwardingRule = errors.New("no forwarding rule found")
)
// Cloud provides GCP cloud metadata information and API access.
type Cloud struct {
forwardingRulesAPI forwardingRulesAPI
imds imdsAPI
instanceAPI instanceAPI
subnetAPI subnetAPI
globalForwardingRulesAPI globalForwardingRulesAPI
regionalForwardingRulesAPI regionalForwardingRulesAPI
imds imdsAPI
instanceAPI instanceAPI
subnetAPI subnetAPI
closers []func() error
}
@ -53,11 +57,19 @@ func New(ctx context.Context) (cloud *Cloud, err error) {
return nil, err
}
closers = append(closers, insAPI.Close)
forwardingRulesAPI, err := compute.NewGlobalForwardingRulesRESTClient(ctx)
globalForwardingRulesAPI, err := compute.NewGlobalForwardingRulesRESTClient(ctx)
if err != nil {
return nil, err
}
closers = append(closers, forwardingRulesAPI.Close)
closers = append(closers, globalForwardingRulesAPI.Close)
regionalForwardingRulesAPI, err := compute.NewForwardingRulesRESTClient(ctx)
if err != nil {
return nil, err
}
closers = append(closers, regionalForwardingRulesAPI.Close)
subnetAPI, err := compute.NewSubnetworksRESTClient(ctx)
if err != nil {
return nil, err
@ -65,11 +77,12 @@ func New(ctx context.Context) (cloud *Cloud, err error) {
closers = append(closers, subnetAPI.Close)
return &Cloud{
imds: imds.NewClient(nil),
instanceAPI: &instanceClient{insAPI},
forwardingRulesAPI: &forwardingRulesClient{forwardingRulesAPI},
subnetAPI: subnetAPI,
closers: closers,
imds: imds.NewClient(nil),
instanceAPI: &instanceClient{insAPI},
globalForwardingRulesAPI: &globalForwardingRulesClient{globalForwardingRulesAPI},
regionalForwardingRulesAPI: &regionalForwardingRulesClient{regionalForwardingRulesAPI},
subnetAPI: subnetAPI,
closers: closers,
}, nil
}
@ -91,8 +104,36 @@ func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
return "", err
}
// First try to find a global forwarding rule.
endpoint, err := c.getGlobalForwardingRule(ctx, project, uid)
if err != nil && !errors.Is(err, errNoForwardingRule) {
return "", fmt.Errorf("getting global forwarding rule: %w", err)
} else if err == nil {
return endpoint, nil
}
// If no global forwarding rule was found, try to find a regional forwarding rule.
region := zoneFromRegionRegex.FindString(zone)
if region == "" {
return "", fmt.Errorf("invalid zone %s", zone)
}
endpoint, err = c.getRegionalForwardingRule(ctx, project, uid, region)
if err != nil && !errors.Is(err, errNoForwardingRule) {
return "", fmt.Errorf("getting regional forwarding rule: %w", err)
} else if err == nil {
return endpoint, nil
}
return "", fmt.Errorf("kubernetes load balancer with UID %s not found: %w", uid, err)
}
// getGlobalForwardingRule returns the endpoint of the load balancer if it is a global load balancer.
// This functions returns ErrNoForwardingRule if no forwarding rule was found.
func (c *Cloud) getGlobalForwardingRule(ctx context.Context, project, uid string) (string, error) {
var resp *computepb.ForwardingRule
iter := c.forwardingRulesAPI.List(ctx, &computepb.ListGlobalForwardingRulesRequest{
var err error
iter := c.globalForwardingRulesAPI.List(ctx, &computepb.ListGlobalForwardingRulesRequest{
Project: project,
Filter: proto.String(fmt.Sprintf("(labels.%s:%s) AND (labels.%s:kubernetes)", cloud.TagUID, uid, tagUsage)),
})
@ -106,8 +147,39 @@ func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
portRange := strings.Split(*resp.PortRange, "-")
return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil
}
if err != iterator.Done {
return "", fmt.Errorf("error listing global forwarding rules with UID %s: %w", uid, err)
}
return "", errNoForwardingRule
}
return "", fmt.Errorf("kubernetes load balancer with UID %s not found: %w", uid, err)
// getRegionalForwardingRule returns the endpoint of the load balancer if it is a regional load balancer.
// This functions returns ErrNoForwardingRule if no forwarding rule was found.
func (c *Cloud) getRegionalForwardingRule(ctx context.Context, project, uid, region string) (string, error) {
var resp *computepb.ForwardingRule
var err error
iter := c.regionalForwardingRulesAPI.List(ctx, &computepb.ListForwardingRulesRequest{
Project: project,
Region: region,
Filter: proto.String(fmt.Sprintf("(labels.%s:%s) AND (labels.%s:kubernetes)", cloud.TagUID, uid, tagUsage)),
})
for resp, err = iter.Next(); err == nil; resp, err = iter.Next() {
if resp.PortRange == nil {
continue
}
if resp.IPAddress == nil {
continue
}
if resp.Region == nil {
continue
}
portRange := strings.Split(*resp.PortRange, "-")
return net.JoinHostPort(*resp.IPAddress, portRange[0]), nil
}
if err != iterator.Done {
return "", fmt.Errorf("error listing global forwarding rules with UID %s: %w", uid, err)
}
return "", errNoForwardingRule
}
// List retrieves all instances belonging to the current constellation.

152
internal/cloud/gcp/cloud_test.go
View file

@ -207,13 +207,14 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
}
testCases := map[string]struct {
imds stubIMDS
instanceAPI stubInstanceAPI
forwardingRulesAPI stubForwardingRulesAPI
wantEndpoint string
wantErr bool
imds stubIMDS
instanceAPI stubInstanceAPI
globalForwardingRulesAPI stubGlobalForwardingRulesAPI
regionalForwardingRulesAPI stubRegionalForwardingRulesAPI
wantEndpoint string
wantErr bool
}{
"success": {
"success global forwarding rule": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
@ -222,7 +223,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
forwardingRulesAPI: stubForwardingRulesAPI{
globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
@ -232,18 +233,49 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
},
},
},
regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
wantEndpoint: "192.0.2.255:6443",
},
"imds error": {
"success regional forwarding rule": {
imds: stubIMDS{
projectIDErr: someErr,
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
forwardingRulesAPI: stubForwardingRulesAPI{
globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
PortRange: proto.String("6443"),
IPAddress: proto.String("192.0.2.255"),
Region: proto.String("someRegion"),
},
},
},
},
wantEndpoint: "192.0.2.255:6443",
},
"regional forwarding rule has no region": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
@ -255,7 +287,31 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
},
wantErr: true,
},
"iterator error": {
"imds error": {
imds: stubIMDS{
projectIDErr: someErr,
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
PortRange: proto.String("6443"),
IPAddress: proto.String("192.0.2.255"),
},
},
},
},
regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
wantErr: true,
},
"global forwarding rule iterator error": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
@ -264,7 +320,29 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
forwardingRulesAPI: stubForwardingRulesAPI{
globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
err: someErr,
},
},
regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
wantErr: true,
},
"regional forwarding rule iterator error": {
imds: stubIMDS{
projectID: "someProject",
zone: "someZone-west3-b",
instanceName: "someInstance",
},
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
err: someErr,
},
@ -280,7 +358,10 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
forwardingRulesAPI: stubForwardingRulesAPI{
globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
wantErr: true,
@ -294,7 +375,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
forwardingRulesAPI: stubForwardingRulesAPI{
globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
@ -303,6 +384,9 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
},
},
},
regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
wantErr: true,
},
"missing IP address": {
@ -314,7 +398,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
instanceAPI: stubInstanceAPI{
instance: goodInstance,
},
forwardingRulesAPI: stubForwardingRulesAPI{
globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
@ -323,6 +407,9 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
},
},
},
regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
wantErr: true,
},
"get instance error": {
@ -334,7 +421,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
instanceAPI: stubInstanceAPI{
instanceErr: someErr,
},
forwardingRulesAPI: stubForwardingRulesAPI{
globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
@ -344,6 +431,9 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
},
},
},
regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
wantErr: true,
},
"invalid instance": {
@ -355,7 +445,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
instanceAPI: stubInstanceAPI{
instance: nil,
},
forwardingRulesAPI: stubForwardingRulesAPI{
globalForwardingRulesAPI: stubGlobalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{
forwardingRules: []*computepb.ForwardingRule{
{
@ -365,6 +455,9 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
},
},
},
regionalForwardingRulesAPI: stubRegionalForwardingRulesAPI{
iterator: &stubForwardingRulesIterator{},
},
wantErr: true,
},
}
@ -374,9 +467,10 @@ func TestGetLoadbalancerEndpoint(t *testing.T) {
assert := assert.New(t)
cloud := &Cloud{
imds: &tc.imds,
instanceAPI: &tc.instanceAPI,
forwardingRulesAPI: &tc.forwardingRulesAPI,
imds: &tc.imds,
instanceAPI: &tc.instanceAPI,
globalForwardingRulesAPI: &tc.globalForwardingRulesAPI,
regionalForwardingRulesAPI: &tc.regionalForwardingRulesAPI,
}
endpoint, err := cloud.GetLoadBalancerEndpoint(context.Background())
@ -861,17 +955,29 @@ func TestInitSecretHash(t *testing.T) {
}
}
type stubForwardingRulesAPI struct {
type stubGlobalForwardingRulesAPI struct {
iterator forwardingRuleIterator
}
func (s *stubForwardingRulesAPI) List(
func (s *stubGlobalForwardingRulesAPI) List(
ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption,
) forwardingRuleIterator {
return s.iterator
}
func (s *stubForwardingRulesAPI) Close() error { return nil }
func (s *stubGlobalForwardingRulesAPI) Close() error { return nil }
type stubRegionalForwardingRulesAPI struct {
iterator forwardingRuleIterator
}
func (s *stubRegionalForwardingRulesAPI) List(
ctx context.Context, req *computepb.ListForwardingRulesRequest, opts ...gax.CallOption,
) forwardingRuleIterator {
return s.iterator
}
func (s *stubRegionalForwardingRulesAPI) Close() error { return nil }
type stubForwardingRulesIterator struct {
ctr int

7
internal/cloud/gcp/interface.go
View file

@ -13,11 +13,16 @@ import (
"github.com/googleapis/gax-go/v2"
)
type forwardingRulesAPI interface {
type globalForwardingRulesAPI interface {
List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest, opts ...gax.CallOption) forwardingRuleIterator
Close() error
}
type regionalForwardingRulesAPI interface {
List(ctx context.Context, req *computepb.ListForwardingRulesRequest, opts ...gax.CallOption) forwardingRuleIterator
Close() error
}
type imdsAPI interface {
InstanceID() (string, error)
ProjectID() (string, error)

20
internal/cloud/gcp/wrappers.go
View file

@ -22,20 +22,34 @@ type instanceIterator interface {
Next() (*computepb.Instance, error)
}
type forwardingRulesClient struct {
type globalForwardingRulesClient struct {
*compute.GlobalForwardingRulesClient
}
func (c *forwardingRulesClient) Close() error {
func (c *globalForwardingRulesClient) Close() error {
return c.GlobalForwardingRulesClient.Close()
}
func (c *forwardingRulesClient) List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest,
func (c *globalForwardingRulesClient) List(ctx context.Context, req *computepb.ListGlobalForwardingRulesRequest,
opts ...gax.CallOption,
) forwardingRuleIterator {
return c.GlobalForwardingRulesClient.List(ctx, req)
}
type regionalForwardingRulesClient struct {
*compute.ForwardingRulesClient
}
func (c *regionalForwardingRulesClient) Close() error {
return c.ForwardingRulesClient.Close()
}
func (c *regionalForwardingRulesClient) List(ctx context.Context, req *computepb.ListForwardingRulesRequest,
opts ...gax.CallOption,
) forwardingRuleIterator {
return c.ForwardingRulesClient.List(ctx, req)
}
type instanceClient struct {
*compute.InstancesClient
}