mirror of
https://github.com/edgelesssys/constellation.git
synced 2025-09-19 04:24:39 -04:00
bootstrapper: add fallback endpoint and custom endpoint to SAN field (#2108)
terraform: collect apiserver cert SANs and support custom endpoint constants: add new constants for cluster configuration and custom endpoint cloud: support apiserver cert sans and prepare for endpoint migration on AWS config: add customEndpoint field bootstrapper: use per-CSP apiserver cert SANs cli: route customEndpoint to terraform and add migration for apiserver cert SANs bootstrapper: change interface of GetLoadBalancerEndpoint to return host and port separately
This commit is contained in:
Malte Poll
GitHub
parent
3324a4eba2
commit
8da6a23aa5
64 changed files with 724 additions and 301 deletions
|
|
@ -12,6 +12,7 @@ go_library(
|
|||
deps = [
|
||||
"//internal/cloud",
|
||||
"//internal/cloud/metadata",
|
||||
"//internal/constants",
|
||||
"//internal/role",
|
||||
"@com_github_aws_aws_sdk_go_v2//aws",
|
||||
"@com_github_aws_aws_sdk_go_v2_config//:config",
|
||||
|
|
@ -21,6 +22,7 @@ go_library(
|
|||
"@com_github_aws_aws_sdk_go_v2_service_ec2//:ec2",
|
||||
"@com_github_aws_aws_sdk_go_v2_service_ec2//types",
|
||||
"@com_github_aws_aws_sdk_go_v2_service_elasticloadbalancingv2//:elasticloadbalancingv2",
|
||||
"@com_github_aws_aws_sdk_go_v2_service_elasticloadbalancingv2//types",
|
||||
"@com_github_aws_aws_sdk_go_v2_service_resourcegroupstaggingapi//:resourcegroupstaggingapi",
|
||||
"@com_github_aws_aws_sdk_go_v2_service_resourcegroupstaggingapi//types",
|
||||
"@io_k8s_utils//clock",
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
|
|
@ -26,10 +27,12 @@ import (
|
|||
"github.com/aws/aws-sdk-go-v2/service/ec2"
|
||||
ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
|
||||
"github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2"
|
||||
elasticloadbalancingv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
|
||||
"github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi"
|
||||
tagType "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types"
|
||||
"github.com/edgelesssys/constellation/v2/internal/cloud"
|
||||
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
|
||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
"github.com/edgelesssys/constellation/v2/internal/role"
|
||||
)
|
||||
|
||||
|
|
@ -48,6 +51,7 @@ type loadbalancerAPI interface {
|
|||
|
||||
type ec2API interface {
|
||||
DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error)
|
||||
DescribeAddresses(context.Context, *ec2.DescribeAddressesInput, ...func(*ec2.Options)) (*ec2.DescribeAddressesOutput, error)
|
||||
}
|
||||
|
||||
type imdsAPI interface {
|
||||
|
|
@ -126,46 +130,100 @@ func (c *Cloud) InitSecretHash(ctx context.Context) ([]byte, error) {
|
|||
}
|
||||
|
||||
// GetLoadBalancerEndpoint returns the endpoint of the load balancer.
|
||||
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
|
||||
// TODO(malt3): remove old infrastructure code once we have migrated to using DNS as the load balancer endpoint.
|
||||
func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) {
|
||||
// try new architecture first
|
||||
uid, err := c.readInstanceTag(ctx, cloud.TagUID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("retrieving uid tag: %w", err)
|
||||
return "", "", fmt.Errorf("retrieving uid tag: %w", err)
|
||||
}
|
||||
describeIPsOutput, err := c.ec2.DescribeAddresses(ctx, &ec2.DescribeAddressesInput{
|
||||
Filters: []ec2Types.Filter{
|
||||
{
|
||||
Name: aws.String(cloud.TagUID),
|
||||
Values: []string{uid},
|
||||
},
|
||||
{
|
||||
Name: aws.String("constellation-ip-endpoint"),
|
||||
Values: []string{"legacy-primary-zone"},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err == nil && len(describeIPsOutput.Addresses) == 1 && describeIPsOutput.Addresses[0].PublicIp != nil {
|
||||
return *describeIPsOutput.Addresses[0].PublicIp, strconv.FormatInt(constants.KubernetesPort, 10), nil
|
||||
}
|
||||
// fallback to old architecture
|
||||
// this will be removed in the future
|
||||
hostname, err := c.getLoadBalancerIPOldInfrastructure(ctx)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("retrieving load balancer ip: %w", err)
|
||||
}
|
||||
return hostname, strconv.FormatInt(constants.KubernetesPort, 10), nil
|
||||
}
|
||||
|
||||
// getLoadBalancerIPOldInfrastructure returns the IP of the load balancer.
|
||||
// This is only used for the old infrastructure.
|
||||
// This will be removed in the future.
|
||||
func (c *Cloud) getLoadBalancerIPOldInfrastructure(ctx context.Context) (string, error) {
|
||||
loadbalancer, err := c.getLoadBalancer(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("finding Constellation load balancer: %w", err)
|
||||
}
|
||||
|
||||
// TODO(malt3): Add support for multiple availability zones in the lb frontend.
|
||||
// This can only be done after we have migrated to using DNS as the load balancer endpoint.
|
||||
// At that point, we don't need to care about the number of availability zones anymore.
|
||||
if len(loadbalancer.AvailabilityZones) != 1 {
|
||||
return "", fmt.Errorf("%d availability zones found; expected 1", len(loadbalancer.AvailabilityZones))
|
||||
}
|
||||
|
||||
if len(loadbalancer.AvailabilityZones[0].LoadBalancerAddresses) != 1 {
|
||||
return "", fmt.Errorf("%d load balancer addresses found; expected 1", len(loadbalancer.AvailabilityZones[0].LoadBalancerAddresses))
|
||||
}
|
||||
if loadbalancer.AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress == nil {
|
||||
return "", errors.New("load balancer address is nil")
|
||||
}
|
||||
|
||||
return *loadbalancer.AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress, nil
|
||||
}
|
||||
|
||||
/*
|
||||
// TODO(malt3): uncomment and use as soon as we switch the primary endpoint to DNS.
|
||||
func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) {
|
||||
loadbalancer, err := c.getLoadBalancer(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("finding Constellation load balancer: %w", err)
|
||||
}
|
||||
if loadbalancer.DNSName == nil {
|
||||
return "", errors.New("load balancer dns name missing")
|
||||
}
|
||||
return *loadbalancer.DNSName, nil
|
||||
}
|
||||
*/
|
||||
|
||||
func (c *Cloud) getLoadBalancer(ctx context.Context) (*elasticloadbalancingv2types.LoadBalancer, error) {
|
||||
uid, err := c.readInstanceTag(ctx, cloud.TagUID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("retrieving uid tag: %w", err)
|
||||
}
|
||||
arns, err := c.getARNsByTag(ctx, uid, "elasticloadbalancing:loadbalancer")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("retrieving load balancer ARNs: %w", err)
|
||||
return nil, fmt.Errorf("retrieving load balancer ARNs: %w", err)
|
||||
}
|
||||
if len(arns) != 1 {
|
||||
return "", fmt.Errorf("%d load balancers found", len(arns))
|
||||
return nil, fmt.Errorf("%d load balancers found", len(arns))
|
||||
}
|
||||
|
||||
output, err := c.loadbalancer.DescribeLoadBalancers(ctx, &elasticloadbalancingv2.DescribeLoadBalancersInput{
|
||||
LoadBalancerArns: arns,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("retrieving load balancer: %w", err)
|
||||
return nil, fmt.Errorf("retrieving load balancer: %w", err)
|
||||
}
|
||||
if len(output.LoadBalancers) != 1 {
|
||||
return "", fmt.Errorf("%d load balancers found; expected 1", len(output.LoadBalancers))
|
||||
return nil, fmt.Errorf("%d load balancers found; expected 1", len(output.LoadBalancers))
|
||||
}
|
||||
|
||||
// TODO(malt3): Add support for multiple availability zones in the lb frontend.
|
||||
// This can only be done after we have migrated to using DNS as the load balancer endpoint.
|
||||
// At that point, we don't need to care about the number of availability zones anymore.
|
||||
if len(output.LoadBalancers[0].AvailabilityZones) != 1 {
|
||||
return "", fmt.Errorf("%d availability zones found; expected 1", len(output.LoadBalancers[0].AvailabilityZones))
|
||||
}
|
||||
|
||||
if len(output.LoadBalancers[0].AvailabilityZones[0].LoadBalancerAddresses) != 1 {
|
||||
return "", fmt.Errorf("%d load balancer addresses found; expected 1", len(output.LoadBalancers[0].AvailabilityZones[0].LoadBalancerAddresses))
|
||||
}
|
||||
if output.LoadBalancers[0].AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress == nil {
|
||||
return "", errors.New("load balancer address is nil")
|
||||
}
|
||||
|
||||
// TODO(malt3): ideally, we would use DNS here instead of IP addresses.
|
||||
// Requires changes to the infrastructure.
|
||||
return *output.LoadBalancers[0].AvailabilityZones[0].LoadBalancerAddresses[0].IpAddress, nil
|
||||
return &output.LoadBalancers[0], nil
|
||||
}
|
||||
|
||||
// getARNsByTag returns a list of ARNs that have the given tag.
|
||||
|
|
|
|||
|
|
@ -485,7 +485,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
|
|||
ec2API *stubEC2
|
||||
loadbalancer *stubLoadbalancer
|
||||
resourceapi *stubResourceGroupTagging
|
||||
wantAddr string
|
||||
wantHost string
|
||||
wantErr bool
|
||||
}{
|
||||
"success retrieving loadbalancer endpoint": {
|
||||
|
|
@ -514,6 +514,47 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
describeAddressesResp: &ec2.DescribeAddressesOutput{
|
||||
Addresses: []ec2Types.Address{
|
||||
{
|
||||
Tags: []ec2Types.Tag{
|
||||
{Key: aws.String(cloud.TagUID), Value: aws.String("uid")},
|
||||
{Key: aws.String("constellation-ip-endpoint"), Value: aws.String("legacy-primary-zone")},
|
||||
},
|
||||
PublicIp: aws.String(lbAddr),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantHost: lbAddr,
|
||||
},
|
||||
"success retrieving loadbalancer endpoint legacy": {
|
||||
imds: &stubIMDS{
|
||||
instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
|
||||
InstanceIdentityDocument: imds.InstanceIdentityDocument{
|
||||
InstanceID: "test-instance-id",
|
||||
},
|
||||
},
|
||||
},
|
||||
ec2API: &stubEC2{
|
||||
selfInstance: &ec2.DescribeInstancesOutput{
|
||||
Reservations: []ec2Types.Reservation{
|
||||
{
|
||||
Instances: []ec2Types.Instance{
|
||||
{
|
||||
InstanceId: aws.String("test-instance-id"),
|
||||
Tags: []ec2Types.Tag{
|
||||
{
|
||||
Key: aws.String(cloud.TagUID),
|
||||
Value: aws.String("uid"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
describeAddressesErr: errors.New("using legacy infrastructure"),
|
||||
},
|
||||
loadbalancer: &stubLoadbalancer{
|
||||
describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
|
||||
|
|
@ -541,9 +582,9 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
wantAddr: lbAddr,
|
||||
wantHost: lbAddr,
|
||||
},
|
||||
"too many ARNs": {
|
||||
"too many ARNs legacy": {
|
||||
imds: &stubIMDS{
|
||||
instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
|
||||
InstanceIdentityDocument: imds.InstanceIdentityDocument{
|
||||
|
|
@ -569,6 +610,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
describeAddressesErr: errors.New("using legacy infrastructure"),
|
||||
},
|
||||
loadbalancer: &stubLoadbalancer{
|
||||
describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
|
||||
|
|
@ -601,7 +643,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
|
|||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"too many ARNs (paged)": {
|
||||
"too many ARNs (paged) legacy": {
|
||||
imds: &stubIMDS{
|
||||
instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
|
||||
InstanceIdentityDocument: imds.InstanceIdentityDocument{
|
||||
|
|
@ -627,6 +669,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
describeAddressesErr: errors.New("using legacy infrastructure"),
|
||||
},
|
||||
loadbalancer: &stubLoadbalancer{
|
||||
describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
|
||||
|
|
@ -664,7 +707,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
|
|||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"loadbalancer has no availability zones": {
|
||||
"loadbalancer has no availability zones legacy": {
|
||||
imds: &stubIMDS{
|
||||
instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
|
||||
InstanceIdentityDocument: imds.InstanceIdentityDocument{
|
||||
|
|
@ -690,6 +733,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
describeAddressesErr: errors.New("using legacy infrastructure"),
|
||||
},
|
||||
loadbalancer: &stubLoadbalancer{
|
||||
describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
|
||||
|
|
@ -711,7 +755,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
|
|||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"failure to get resources by tag": {
|
||||
"failure to get resources by tag legacy": {
|
||||
imds: &stubIMDS{
|
||||
instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{
|
||||
InstanceIdentityDocument: imds.InstanceIdentityDocument{
|
||||
|
|
@ -737,6 +781,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
describeAddressesErr: errors.New("using legacy infrastructure"),
|
||||
},
|
||||
loadbalancer: &stubLoadbalancer{
|
||||
describeLoadBalancersOut: &elasticloadbalancingv2.DescribeLoadBalancersOutput{
|
||||
|
|
@ -772,14 +817,15 @@ func TestGetLoadBalancerEndpoint(t *testing.T) {
|
|||
resourceapiClient: tc.resourceapi,
|
||||
}
|
||||
|
||||
endpoint, err := m.GetLoadBalancerEndpoint(context.Background())
|
||||
gotHost, gotPort, err := m.GetLoadBalancerEndpoint(context.Background())
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
assert.NoError(err)
|
||||
assert.Equal(tc.wantAddr, endpoint)
|
||||
assert.Equal(tc.wantHost, gotHost)
|
||||
assert.Equal("6443", gotPort)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -973,6 +1019,8 @@ type stubEC2 struct {
|
|||
selfInstance *ec2.DescribeInstancesOutput
|
||||
describeInstancesResp1 *ec2.DescribeInstancesOutput
|
||||
describeInstancesResp2 *ec2.DescribeInstancesOutput
|
||||
describeAddressesErr error
|
||||
describeAddressesResp *ec2.DescribeAddressesOutput
|
||||
}
|
||||
|
||||
func (s *stubEC2) DescribeInstances(_ context.Context, in *ec2.DescribeInstancesInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) {
|
||||
|
|
@ -985,6 +1033,10 @@ func (s *stubEC2) DescribeInstances(_ context.Context, in *ec2.DescribeInstances
|
|||
return s.describeInstancesResp2, s.describeInstancesErr
|
||||
}
|
||||
|
||||
func (s *stubEC2) DescribeAddresses(context.Context, *ec2.DescribeAddressesInput, ...func(*ec2.Options)) (*ec2.DescribeAddressesOutput, error) {
|
||||
return s.describeAddressesResp, s.describeAddressesErr
|
||||
}
|
||||
|
||||
type stubLoadbalancer struct {
|
||||
describeLoadBalancersErr error
|
||||
describeLoadBalancersOut *elasticloadbalancingv2.DescribeLoadBalancersOutput
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue