Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-11-11 08:15:01 -05:00
Code Issues Projects Releases Packages Wiki Activity

Use multiple loadbalancers on GCP

Browse source
This commit is contained in:
katexochen 2022-08-01 16:51:34 +02:00 • committed by Paul Meyer
parent c954ec089f
commit a02a46e454
59 changed files with 1629 additions and 557 deletions
Download patch file Download diff file Expand all files Collapse all files

7
bootstrapper/cloudprovider/azure/metadata.go
View file

@ -247,8 +247,11 @@ func (m *Metadata) GetLoadBalancerName(ctx context.Context) (string, error) {
return *lb.Name, nil
}
// GetLoadBalancerIP retrieves the first load balancer IP from cloud provider metadata.
func (m *Metadata) GetLoadBalancerIP(ctx context.Context) (string, error) {
// GetLoadBalancerEndpoint retrieves the first load balancer IP from cloud provider metadata.
//
// The returned string is an IP address without a port, but the method name needs to satisfy the
// metadata interface.
func (m *Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
lb, err := m.getLoadBalancer(ctx)
if err != nil {
return "", err

6
bootstrapper/cloudprovider/azure/metadata_test.go
View file

@ -307,7 +307,7 @@ func TestGetLoadBalancerName(t *testing.T) {
}
}
func TestGetLoadBalancerIP(t *testing.T) {
func TestGetLoadBalancerEndpoint(t *testing.T) {
loadBalancerName := "load-balancer-name"
publicIP := "192.0.2.1"
correctPublicIPID := "/subscriptions/subscription/resourceGroups/resourceGroup/providers/Microsoft.Network/publicIPAddresses/pubIPName"
@ -319,7 +319,7 @@ func TestGetLoadBalancerIP(t *testing.T) {
wantIP string
wantErr bool
}{
"GetLoadBalancerIP works": {
"GetLoadBalancerEndpoint works": {
imdsAPI: newScaleSetIMDSStub(),
loadBalancerAPI: &stubLoadBalancersAPI{
pager: &stubLoadBalancersClientListPager{
@ -446,7 +446,7 @@ func TestGetLoadBalancerIP(t *testing.T) {
loadBalancerAPI: tc.loadBalancerAPI,
publicIPAddressesAPI: tc.publicIPAddressesAPI,
}
loadbalancerName, err := metadata.GetLoadBalancerIP(context.Background())
loadbalancerName, err := metadata.GetLoadBalancerEndpoint(context.Background())
if tc.wantErr {
assert.Error(err)
return

13
bootstrapper/cloudprovider/gcp/client.go
View file

@ -2,7 +2,9 @@ package gcp
import (
"context"
"errors"
"fmt"
"net"
"regexp"
"strings"
@ -213,8 +215,8 @@ func (c *Client) RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone,
return *(subnetwork.SecondaryIpRanges[0]).IpCidrRange, nil
}
// RetrieveLoadBalancerIP returns the IP address of the load balancer specified by project, zone and loadBalancerName.
func (c *Client) RetrieveLoadBalancerIP(ctx context.Context, project, zone string) (string, error) {
// RetrieveLoadBalancerEndpoint returns the endpoint of the load balancer with the constellation-uid tag.
func (c *Client) RetrieveLoadBalancerEndpoint(ctx context.Context, project, zone string) (string, error) {
uid, err := c.UID()
if err != nil {
return "", err
@ -226,8 +228,8 @@ func (c *Client) RetrieveLoadBalancerIP(ctx context.Context, project, zone strin
}
req := &computepb.ListForwardingRulesRequest{
Region: region,
Project: project,
Region: region,
}
iter := c.forwardingRulesAPI.List(ctx, req)
for {
@ -239,7 +241,10 @@ func (c *Client) RetrieveLoadBalancerIP(ctx context.Context, project, zone strin
return "", fmt.Errorf("retrieving load balancer IP failed: %w", err)
}
if resp.Labels["constellation-uid"] == uid {
return *resp.IPAddress, nil
if len(resp.Ports) == 0 {
return "", errors.New("load balancer with searched UID has no ports")
}
return net.JoinHostPort(*resp.IPAddress, resp.Ports[0]), nil
}
}

32
bootstrapper/cloudprovider/gcp/client_test.go
View file

@ -773,7 +773,7 @@ func TestRetrieveSubnetworkAliasCIDR(t *testing.T) {
}
}
func TestRetrieveLoadBalancerIP(t *testing.T) {
func TestRetrieveLoadBalancerEndpoint(t *testing.T) {
loadBalancerIP := "192.0.2.1"
uid := "uid"
someErr := errors.New("some error")
@ -783,13 +783,14 @@ func TestRetrieveLoadBalancerIP(t *testing.T) {
wantLoadBalancerIP string
wantErr bool
}{
"RetrieveSubnetworkAliasCIDR works": {
"works": {
stubMetadataClient: stubMetadataClient{InstanceValue: uid},
stubForwardingRulesClient: stubForwardingRulesClient{
ForwardingRuleIterator: &stubForwardingRuleIterator{
rules: []*computepb.ForwardingRule{
{
IPAddress: proto.String(loadBalancerIP),
Ports: []string{"100"},
Labels: map[string]string{"constellation-uid": uid},
},
},
@ -797,20 +798,36 @@ func TestRetrieveLoadBalancerIP(t *testing.T) {
},
wantLoadBalancerIP: loadBalancerIP,
},
"RetrieveSubnetworkAliasCIDR fails when no matching load balancers exists": {
"fails when no matching load balancers exists": {
stubMetadataClient: stubMetadataClient{InstanceValue: uid},
stubForwardingRulesClient: stubForwardingRulesClient{
ForwardingRuleIterator: &stubForwardingRuleIterator{
rules: []*computepb.ForwardingRule{
{
IPAddress: proto.String(loadBalancerIP),
Ports: []string{"100"},
},
},
},
},
wantErr: true,
},
"RetrieveSubnetworkAliasCIDR fails when retrieving uid": {
"fails when retrieving uid": {
stubMetadataClient: stubMetadataClient{InstanceErr: someErr},
stubForwardingRulesClient: stubForwardingRulesClient{
ForwardingRuleIterator: &stubForwardingRuleIterator{
rules: []*computepb.ForwardingRule{
{
IPAddress: proto.String(loadBalancerIP),
Ports: []string{"100"},
Labels: map[string]string{"constellation-uid": uid},
},
},
},
},
wantErr: true,
},
"fails when answer has empty port range": {
stubMetadataClient: stubMetadataClient{InstanceErr: someErr},
stubForwardingRulesClient: stubForwardingRulesClient{
ForwardingRuleIterator: &stubForwardingRuleIterator{
@ -824,7 +841,7 @@ func TestRetrieveLoadBalancerIP(t *testing.T) {
},
wantErr: true,
},
"RetrieveSubnetworkAliasCIDR fails when retrieving loadbalancer IP": {
"fails when retrieving loadbalancer IP": {
stubMetadataClient: stubMetadataClient{},
stubForwardingRulesClient: stubForwardingRulesClient{
ForwardingRuleIterator: &stubForwardingRuleIterator{
@ -832,6 +849,7 @@ func TestRetrieveLoadBalancerIP(t *testing.T) {
rules: []*computepb.ForwardingRule{
{
IPAddress: proto.String(loadBalancerIP),
Ports: []string{"100"},
Labels: map[string]string{"constellation-uid": uid},
},
},
@ -846,14 +864,14 @@ func TestRetrieveLoadBalancerIP(t *testing.T) {
require := require.New(t)
client := Client{forwardingRulesAPI: tc.stubForwardingRulesClient, metadataAPI: tc.stubMetadataClient}
aliasCIDR, err := client.RetrieveLoadBalancerIP(context.Background(), "project", "us-central1-a")
aliasCIDR, err := client.RetrieveLoadBalancerEndpoint(context.Background(), "project", "us-central1-a")
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Equal(tc.wantLoadBalancerIP, aliasCIDR)
assert.Equal(tc.wantLoadBalancerIP+":100", aliasCIDR)
})
}
}

10
bootstrapper/cloudprovider/gcp/metadata.go
View file

@ -26,8 +26,8 @@ type API interface {
RetrieveInstanceName() (string, error)
// RetrieveSubnetworkAliasCIDR retrieves the subnetwork CIDR of the current instance.
RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, instanceName string) (string, error)
// RetrieveLoadBalancerIP retrieves the load balancer IP of the current instance.
RetrieveLoadBalancerIP(ctx context.Context, project, zone string) (string, error)
// RetrieveLoadBalancerEndpoint retrieves the load balancer endpoint of the current instance.
RetrieveLoadBalancerEndpoint(ctx context.Context, project, zone string) (string, error)
// SetInstanceMetadata sets metadata key: value of the instance specified by project, zone and instanceName.
SetInstanceMetadata(ctx context.Context, project, zone, instanceName, key, value string) error
// UnsetInstanceMetadata removes a metadata key-value pair of the instance specified by project, zone and instanceName.
@ -111,8 +111,8 @@ func (m *Metadata) SupportsLoadBalancer() bool {
return true
}
// GetLoadBalancerIP returns the IP of the load balancer.
func (m *Metadata) GetLoadBalancerIP(ctx context.Context) (string, error) {
// GetLoadBalancerEndpoint returns the endpoint of the load balancer.
func (m *Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
project, err := m.api.RetrieveProjectID()
if err != nil {
return "", err
@ -121,7 +121,7 @@ func (m *Metadata) GetLoadBalancerIP(ctx context.Context) (string, error) {
if err != nil {
return "", err
}
return m.api.RetrieveLoadBalancerIP(ctx, project, zone)
return m.api.RetrieveLoadBalancerEndpoint(ctx, project, zone)
}
// UID retrieves the UID of the constellation.

6
bootstrapper/cloudprovider/gcp/metadata_test.go
View file

@ -239,7 +239,7 @@ type stubGCPClient struct {
retrieveInstancesErr error
retrieveInstanceMetadaValues map[string]string
retrieveInstanceMetadataErr error
retrieveSubentworkAliasErr error
retrieveSubnetworkAliasErr error
projectID string
zone string
instanceName string
@ -287,7 +287,7 @@ func (s *stubGCPClient) RetrieveInstanceName() (string, error) {
return s.instanceName, s.retrieveInstanceNameErr
}
func (s *stubGCPClient) RetrieveLoadBalancerIP(ctx context.Context, project, zone string) (string, error) {
func (s *stubGCPClient) RetrieveLoadBalancerEndpoint(ctx context.Context, project, zone string) (string, error) {
return s.loadBalancerIP, s.retrieveLoadBalancerErr
}
@ -315,5 +315,5 @@ func (s *stubGCPClient) UnsetInstanceMetadata(ctx context.Context, project, zone
}
func (s *stubGCPClient) RetrieveSubnetworkAliasCIDR(ctx context.Context, project, zone, instanceName string) (string, error) {
return "", s.retrieveSubentworkAliasErr
return "", s.retrieveSubnetworkAliasErr
}

6
bootstrapper/cloudprovider/qemu/metadata.go
View file

@ -65,9 +65,9 @@ func (m Metadata) SupportsLoadBalancer() bool {
return false
}
// GetLoadBalancerIP returns the IP of the load balancer.
func (m Metadata) GetLoadBalancerIP(ctx context.Context) (string, error) {
panic("function *Metadata.GetLoadBalancerIP not implemented")
// GetLoadBalancerEndpoint returns the endpoint of the load balancer.
func (m Metadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
panic("function *Metadata.GetLoadBalancerEndpoint not implemented")
}
// UID returns the UID of the constellation.

36
bootstrapper/cmd/bootstrapper/main.go
View file

@ -5,13 +5,14 @@ import (
"encoding/json"
"flag"
"io"
"net"
"os"
"strconv"
"strings"
azurecloud "github.com/edgelesssys/constellation/bootstrapper/cloudprovider/azure"
gcpcloud "github.com/edgelesssys/constellation/bootstrapper/cloudprovider/gcp"
qemucloud "github.com/edgelesssys/constellation/bootstrapper/cloudprovider/qemu"
"github.com/edgelesssys/constellation/bootstrapper/internal/joinclient"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/kubectl"
@ -22,7 +23,9 @@ import (
"github.com/edgelesssys/constellation/internal/attestation/qemu"
"github.com/edgelesssys/constellation/internal/attestation/simulator"
"github.com/edgelesssys/constellation/internal/attestation/vtpm"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/iproute"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/oid"
"github.com/spf13/afero"
@ -30,8 +33,6 @@ import (
)
const (
defaultIP = "0.0.0.0"
defaultPort = "9000"
// ConstellationCSP is the environment variable stating which Cloud Service Provider Constellation is running on.
constellationCSP = "CONSTEL_CSP"
)
@ -52,9 +53,10 @@ func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var bindIP, bindPort string
bindIP := "0.0.0.0"
bindPort := strconv.Itoa(constants.BootstrapperPort)
var clusterInitJoiner clusterInitJoiner
var metadataAPI joinclient.MetadataAPI
var metadataAPI metadataAPI
var cloudLogger logging.CloudLogger
var issuer atls.Issuer
var openTPM vtpm.TPMOpenFunc
@ -93,10 +95,12 @@ func main() {
"gcp", k8sapi.NewKubernetesUtil(), &k8sapi.CoreOSConfiguration{}, kubectl.New(), &gcpcloud.CloudControllerManager{},
&gcpcloud.CloudNodeManager{}, &gcpcloud.Autoscaler{}, metadata, pcrsJSON,
)
bindIP = defaultIP
bindPort = defaultPort
openTPM = vtpm.OpenVTPM
fs = afero.NewOsFs()
if err := setLoadbalancerRoute(ctx, metadata); err != nil {
log.With(zap.Error(err)).Fatalf("Failed to set loadbalancer route")
}
log.Infof("Added load balancer IP to routing table")
case "azure":
pcrs, err := vtpm.GetSelectedPCRs(vtpm.OpenVTPM, vtpm.AzurePCRSelection)
if err != nil {
@ -123,8 +127,6 @@ func main() {
&azurecloud.CloudNodeManager{}, &azurecloud.Autoscaler{}, metadata, pcrsJSON,
)
bindIP = defaultIP
bindPort = defaultPort
openTPM = vtpm.OpenVTPM
fs = afero.NewOsFs()
case "qemu":
@ -147,8 +149,6 @@ func main() {
)
metadataAPI = metadata
bindIP = defaultIP
bindPort = defaultPort
openTPM = vtpm.OpenVTPM
fs = afero.NewOsFs()
default:
@ -156,8 +156,6 @@ func main() {
clusterInitJoiner = &clusterFake{}
metadataAPI = &providerMetadataFake{}
cloudLogger = &logging.NopLogger{}
bindIP = defaultIP
bindPort = defaultPort
var simulatedTPMCloser io.Closer
openTPM, simulatedTPMCloser = simulator.NewSimulatedTPMOpenFunc()
defer simulatedTPMCloser.Close()
@ -168,3 +166,15 @@ func main() {
run(issuer, openTPM, fileHandler, clusterInitJoiner, metadataAPI, bindIP, bindPort, log, cloudLogger)
}
func setLoadbalancerRoute(ctx context.Context, meta metadataAPI) error {
endpoint, err := meta.GetLoadBalancerEndpoint(ctx)
if err != nil {
return err
}
ip, _, err := net.SplitHostPort(endpoint)
if err != nil {
return err
}
return iproute.AddToLocalRoutingTable(ctx, ip)
}

8
bootstrapper/cmd/bootstrapper/run.go
View file

@ -1,6 +1,7 @@
package main
import (
"context"
"net"
"github.com/edgelesssys/constellation/bootstrapper/internal/clean"
@ -20,7 +21,7 @@ import (
var version = "0.0.0"
func run(issuer quoteIssuer, tpm vtpm.TPMOpenFunc, fileHandler file.Handler,
kube clusterInitJoiner, metadata joinclient.MetadataAPI,
kube clusterInitJoiner, metadata metadataAPI,
bindIP, bindPort string, log *logger.Logger,
cloudLogger logging.CloudLogger,
) {
@ -90,3 +91,8 @@ type quoteIssuer interface {
// Issue issues a quote for remote attestation for a given message
Issue(userData []byte, nonce []byte) (quote []byte, err error)
}
type metadataAPI interface {
joinclient.MetadataAPI
GetLoadBalancerEndpoint(ctx context.Context) (string, error)
}

4
bootstrapper/cmd/bootstrapper/test.go
View file

@ -55,6 +55,10 @@ func (f *providerMetadataFake) SetVPNIP(ctx context.Context, vpnIP string) error
return nil
}
func (f *providerMetadataFake) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
return "", nil
}
func (f *providerMetadataFake) Supported() bool {
return true
}

6
bootstrapper/internal/initserver/initserver.go
View file

@ -5,6 +5,7 @@ import (
"fmt"
"net"
"strings"
"time"
"github.com/edgelesssys/constellation/bootstrapper/initproto"
"github.com/edgelesssys/constellation/bootstrapper/internal/diskencryption"
@ -21,6 +22,7 @@ import (
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/keepalive"
"google.golang.org/grpc/status"
)
@ -53,12 +55,12 @@ func New(lock locker, kube ClusterInitializer, issuer atls.Issuer, fh file.Handl
grpcServer := grpc.NewServer(
grpc.Creds(atlscredentials.New(issuer, nil)),
grpc.KeepaliveParams(keepalive.ServerParameters{Time: 15 * time.Second}),
log.Named("gRPC").GetServerUnaryInterceptor(),
)
initproto.RegisterAPIServer(grpcServer, server)
server.grpcServer = grpcServer
return server
}
@ -69,6 +71,8 @@ func (s *Server) Serve(ip, port string, cleaner cleaner) error {
if err != nil {
return fmt.Errorf("failed to listen: %w", err)
}
s.log.Infof("Starting")
return s.grpcServer.Serve(lis)
}

12
bootstrapper/internal/kubernetes/cloud_provider.go
View file

@ -21,8 +21,8 @@ type ProviderMetadata interface {
GetSubnetworkCIDR(ctx context.Context) (string, error)
// SupportsLoadBalancer returns true if the cloud provider supports load balancers.
SupportsLoadBalancer() bool
// GetLoadBalancerIP retrieves the load balancer IP.
GetLoadBalancerIP(ctx context.Context) (string, error)
// GetLoadBalancerEndpoint retrieves the load balancer endpoint.
GetLoadBalancerEndpoint(ctx context.Context) (string, error)
// GetInstance retrieves an instance using its providerID.
GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error)
// Supported is used to determine if metadata API is implemented for this cloud provider.
@ -85,8 +85,8 @@ type ClusterAutoscaler interface {
}
type stubProviderMetadata struct {
GetLoadBalancerIPErr error
GetLoadBalancerIPResp string
GetLoadBalancerEndpointErr error
GetLoadBalancerEndpointResp string
GetSubnetworkCIDRErr error
GetSubnetworkCIDRResp string
@ -107,8 +107,8 @@ type stubProviderMetadata struct {
UIDResp string
}
func (m *stubProviderMetadata) GetLoadBalancerIP(ctx context.Context) (string, error) {
return m.GetLoadBalancerIPResp, m.GetLoadBalancerIPErr
func (m *stubProviderMetadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
return m.GetLoadBalancerEndpointResp, m.GetLoadBalancerEndpointErr
}
func (m *stubProviderMetadata) GetSubnetworkCIDR(ctx context.Context) (string, error) {

6
bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go
View file

@ -5,6 +5,7 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubelet"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/versions"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -17,7 +18,6 @@ import (
// Slimmed down to the fields we require
const (
bindPort = 6443
auditLogDir = "/var/log/kubernetes/audit/"
auditLogFile = "audit.log"
auditPolicyPath = "/etc/kubernetes/audit-policy.yaml"
@ -45,7 +45,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sV
},
// AdvertiseAddress will be overwritten later
LocalAPIEndpoint: kubeadm.APIEndpoint{
BindPort: bindPort,
BindPort: constants.KubernetesPort,
},
},
// https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3#ClusterConfiguration
@ -216,7 +216,7 @@ func (k *KubeadmJoinYAML) SetControlPlane(advertiseAddress string) {
k.JoinConfiguration.ControlPlane = &kubeadm.JoinControlPlane{
LocalAPIEndpoint: kubeadm.APIEndpoint{
AdvertiseAddress: advertiseAddress,
BindPort: 6443,
BindPort: constants.KubernetesPort,
},
}
k.JoinConfiguration.SkipPhases = []string{"control-plane-prepare/download-certs"}

81
bootstrapper/internal/kubernetes/kubernetes.go
View file

@ -3,10 +3,8 @@ package kubernetes
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"os/exec"
"strings"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi"
@ -15,6 +13,7 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/util"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/iproute"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/spf13/afero"
@ -93,7 +92,7 @@ func (k *KubeWrapper) InitCluster(
var publicIP string
var nodePodCIDR string
var subnetworkPodCIDR string
var controlPlaneEndpointIP string // this is the IP in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>" hence the unfortunate name
var controlPlaneEndpoint string // this is the endpoint in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>"
var nodeIP string
var validIPs []net.IP
@ -102,7 +101,7 @@ func (k *KubeWrapper) InitCluster(
log.Infof("Retrieving node metadata")
instance, err = k.providerMetadata.Self(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving own instance metadata failed: %w", err)
return nil, fmt.Errorf("retrieving own instance metadata: %w", err)
}
if instance.VPCIP != "" {
validIPs = append(validIPs, net.ParseIP(instance.VPCIP))
@ -120,18 +119,13 @@ func (k *KubeWrapper) InitCluster(
}
subnetworkPodCIDR, err = k.providerMetadata.GetSubnetworkCIDR(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving subnetwork CIDR failed: %w", err)
return nil, fmt.Errorf("retrieving subnetwork CIDR: %w", err)
}
controlPlaneEndpointIP = publicIP
controlPlaneEndpoint = publicIP
if k.providerMetadata.SupportsLoadBalancer() {
controlPlaneEndpointIP, err = k.providerMetadata.GetLoadBalancerIP(ctx)
controlPlaneEndpoint, err = k.providerMetadata.GetLoadBalancerEndpoint(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving load balancer IP failed: %w", err)
}
if k.cloudProvider == "gcp" {
if err := manuallySetLoadbalancerIP(ctx, controlPlaneEndpointIP); err != nil {
return nil, fmt.Errorf("setting load balancer IP failed: %w", err)
}
return nil, fmt.Errorf("retrieving load balancer endpoint: %w", err)
}
}
}
@ -139,7 +133,7 @@ func (k *KubeWrapper) InitCluster(
zap.String("nodeName", nodeName),
zap.String("providerID", providerID),
zap.String("nodeIP", nodeIP),
zap.String("controlPlaneEndpointIP", controlPlaneEndpointIP),
zap.String("controlPlaneEndpointEndpoint", controlPlaneEndpoint),
zap.String("podCIDR", subnetworkPodCIDR),
).Infof("Setting information for node")
@ -149,7 +143,7 @@ func (k *KubeWrapper) InitCluster(
initConfig.SetCertSANs([]string{publicIP, nodeIP})
initConfig.SetNodeName(nodeName)
initConfig.SetProviderID(providerID)
initConfig.SetControlPlaneEndpoint(controlPlaneEndpointIP)
initConfig.SetControlPlaneEndpoint(controlPlaneEndpoint)
initConfigYAML, err := initConfig.Marshal()
if err != nil {
return nil, fmt.Errorf("encoding kubeadm init configuration as YAML: %w", err)
@ -249,15 +243,20 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
}
nodeName := nodeInternalIP
var providerID string
var loadbalancerEndpoint string
if k.providerMetadata.Supported() {
log.Infof("Retrieving node metadata")
instance, err := k.providerMetadata.Self(ctx)
if err != nil {
return fmt.Errorf("retrieving own instance metadata failed: %w", err)
return fmt.Errorf("retrieving own instance metadata: %w", err)
}
providerID = instance.ProviderID
nodeName = instance.Name
nodeInternalIP = instance.VPCIP
loadbalancerEndpoint, err = k.providerMetadata.GetLoadBalancerEndpoint(ctx)
if err != nil {
return fmt.Errorf("retrieving loadbalancer endpoint: %w", err)
}
}
nodeName = k8sCompliantHostname(nodeName)
@ -267,8 +266,19 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
zap.String("nodeIP", nodeInternalIP),
).Infof("Setting information for node")
// Step 2: configure kubeadm join config
// Step 2: Remove load balancer from local routing table on GCP.
if k.cloudProvider == "gcp" {
ip, _, err := net.SplitHostPort(loadbalancerEndpoint)
if err != nil {
return fmt.Errorf("parsing load balancer IP: %w", err)
}
if err := iproute.RemoveFromLocalRoutingTable(ctx, ip); err != nil {
return fmt.Errorf("removing load balancer IP from routing table: %w", err)
}
log.Infof("Removed load balancer IP from routing table")
}
// Step 3: configure kubeadm join config
joinConfig := k.configProvider.JoinConfiguration(k.cloudControllerManager.Supported())
joinConfig.SetAPIServerEndpoint(args.APIServerEndpoint)
joinConfig.SetToken(args.Token)
@ -319,15 +329,15 @@ func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServ
}
ccmConfigMaps, err := k.cloudControllerManager.ConfigMaps(instance)
if err != nil {
return fmt.Errorf("defining ConfigMaps for CCM failed: %w", err)
return fmt.Errorf("defining ConfigMaps for CCM: %w", err)
}
ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance.ProviderID, cloudServiceAccountURI)
if err != nil {
return fmt.Errorf("defining Secrets for CCM failed: %w", err)
return fmt.Errorf("defining Secrets for CCM: %w", err)
}
ccmImage, err := k.cloudControllerManager.Image(k8sVersion)
if err != nil {
return fmt.Errorf("defining Image for CCM failed: %w", err)
return fmt.Errorf("defining Image for CCM: %w", err)
}
cloudControllerManagerConfiguration := resources.NewDefaultCloudControllerManagerDeployment(
@ -335,7 +345,7 @@ func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServ
k.cloudControllerManager.ExtraArgs(), k.cloudControllerManager.Volumes(), k.cloudControllerManager.VolumeMounts(), k.cloudControllerManager.Env(),
)
if err := k.clusterUtil.SetupCloudControllerManager(k.client, cloudControllerManagerConfiguration, ccmConfigMaps, ccmSecrets); err != nil {
return fmt.Errorf("failed to setup cloud-controller-manager: %w", err)
return fmt.Errorf("setting up cloud-controller-manager: %w", err)
}
return nil
@ -347,14 +357,14 @@ func (k *KubeWrapper) setupCloudNodeManager(k8sVersion versions.ValidK8sVersion)
}
nodeManagerImage, err := k.cloudNodeManager.Image(k8sVersion)
if err != nil {
return fmt.Errorf("defining Image for Node Manager failed: %w", err)
return fmt.Errorf("defining Image for Node Manager: %w", err)
}
cloudNodeManagerConfiguration := resources.NewDefaultCloudNodeManagerDeployment(
nodeManagerImage, k.cloudNodeManager.Path(), k.cloudNodeManager.ExtraArgs(),
)
if err := k.clusterUtil.SetupCloudNodeManager(k.client, cloudNodeManagerConfiguration); err != nil {
return fmt.Errorf("failed to setup cloud-node-manager: %w", err)
return fmt.Errorf("setting up cloud-node-manager: %w", err)
}
return nil
@ -366,13 +376,13 @@ func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata,
}
caSecrets, err := k.clusterAutoscaler.Secrets(instance.ProviderID, cloudServiceAccountURI)
if err != nil {
return fmt.Errorf("defining Secrets for cluster-autoscaler failed: %w", err)
return fmt.Errorf("defining Secrets for cluster-autoscaler: %w", err)
}
clusterAutoscalerConfiguration := resources.NewDefaultAutoscalerDeployment(k.clusterAutoscaler.Volumes(), k.clusterAutoscaler.VolumeMounts(), k.clusterAutoscaler.Env(), k8sVersion)
clusterAutoscalerConfiguration.SetAutoscalerCommand(k.clusterAutoscaler.Name(), autoscalingNodeGroups)
if err := k.clusterUtil.SetupAutoscaling(k.client, clusterAutoscalerConfiguration, caSecrets); err != nil {
return fmt.Errorf("failed to setup cluster-autoscaler: %w", err)
return fmt.Errorf("setting up cluster-autoscaler: %w", err)
}
return nil
@ -425,27 +435,6 @@ func (k *KubeWrapper) setupOperators(ctx context.Context) error {
return nil
}
// manuallySetLoadbalancerIP sets the loadbalancer IP of the first control plane during init.
// The GCP guest agent does this usually, but is deployed in the cluster that doesn't exist
// at this point. This is a workaround to set the loadbalancer IP manually, so kubeadm and kubelet
// can talk to the local Kubernetes API server using the loadbalancer IP.
func manuallySetLoadbalancerIP(ctx context.Context, ip string) error {
// https://github.com/GoogleCloudPlatform/guest-agent/blob/792fce795218633bcbde505fb3457a0b24f26d37/google_guest_agent/addresses.go#L179
if !strings.Contains(ip, "/") {
ip = ip + "/32"
}
args := []string{"route", "add", "to", "local", ip, "scope", "host", "dev", "ens3", "proto", "66"}
_, err := exec.CommandContext(ctx, "ip", args...).Output()
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return fmt.Errorf("ip route add (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
}
return fmt.Errorf("ip route add: %w", err)
}
return nil
}
// k8sCompliantHostname transforms a hostname to an RFC 1123 compliant, lowercase subdomain as required by Kubernetes node names.
// The following regex is used by k8s for validation: /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/ .
// Only a simple heuristic is used for now (to lowercase, replace underscores).

16
bootstrapper/internal/kubernetes/kubernetes_test.go
View file

@ -5,12 +5,14 @@ import (
"errors"
"net"
"regexp"
"strconv"
"testing"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/bootstrapper/role"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/stretchr/testify/assert"
@ -86,8 +88,8 @@ func TestInitCluster(t *testing.T) {
PublicIP: publicIP,
AliasIPRanges: []string{aliasIPRange},
},
GetLoadBalancerIPResp: loadbalancerIP,
SupportsLoadBalancerResp: true,
GetLoadBalancerEndpointResp: loadbalancerIP,
SupportsLoadBalancerResp: true,
},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{SupportedResp: false},
@ -148,9 +150,9 @@ func TestInitCluster(t *testing.T) {
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{
GetLoadBalancerIPErr: someErr,
SupportsLoadBalancerResp: true,
SupportedResp: true,
GetLoadBalancerEndpointErr: someErr,
SupportsLoadBalancerResp: true,
SupportedResp: true,
},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{},
@ -319,7 +321,7 @@ func TestInitCluster(t *testing.T) {
func TestJoinCluster(t *testing.T) {
someErr := errors.New("failed")
joinCommand := &kubeadm.BootstrapTokenDiscovery{
APIServerEndpoint: "192.0.2.0:6443",
APIServerEndpoint: "192.0.2.0:" + strconv.Itoa(constants.KubernetesPort),
Token: "kube-fake-token",
CACertHashes: []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"},
}
@ -419,7 +421,7 @@ func TestJoinCluster(t *testing.T) {
ControlPlane: &kubeadm.JoinControlPlane{
LocalAPIEndpoint: kubeadm.APIEndpoint{
AdvertiseAddress: "192.0.2.1",
BindPort: 6443,
BindPort: constants.KubernetesPort,
},
},
SkipPhases: []string{"control-plane-prepare/download-certs"},