Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-09-18 20:14:48 -04:00
Code Issues Projects Releases Packages Wiki Activity

Use multiple loadbalancers on GCP

Browse source
This commit is contained in:
katexochen 2022-08-01 16:51:34 +02:00 • committed by Paul Meyer
parent c954ec089f
commit a02a46e454
59 changed files with 1629 additions and 557 deletions
Download patch file Download diff file Expand all files Collapse all files

6
bootstrapper/internal/initserver/initserver.go
View file

@ -5,6 +5,7 @@ import (
"fmt"
"net"
"strings"
"time"
"github.com/edgelesssys/constellation/bootstrapper/initproto"
"github.com/edgelesssys/constellation/bootstrapper/internal/diskencryption"
@ -21,6 +22,7 @@ import (
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/keepalive"
"google.golang.org/grpc/status"
)
@ -53,12 +55,12 @@ func New(lock locker, kube ClusterInitializer, issuer atls.Issuer, fh file.Handl
grpcServer := grpc.NewServer(
grpc.Creds(atlscredentials.New(issuer, nil)),
grpc.KeepaliveParams(keepalive.ServerParameters{Time: 15 * time.Second}),
log.Named("gRPC").GetServerUnaryInterceptor(),
)
initproto.RegisterAPIServer(grpcServer, server)
server.grpcServer = grpcServer
return server
}
@ -69,6 +71,8 @@ func (s *Server) Serve(ip, port string, cleaner cleaner) error {
if err != nil {
return fmt.Errorf("failed to listen: %w", err)
}
s.log.Infof("Starting")
return s.grpcServer.Serve(lis)
}

12
bootstrapper/internal/kubernetes/cloud_provider.go
View file

@ -21,8 +21,8 @@ type ProviderMetadata interface {
GetSubnetworkCIDR(ctx context.Context) (string, error)
// SupportsLoadBalancer returns true if the cloud provider supports load balancers.
SupportsLoadBalancer() bool
// GetLoadBalancerIP retrieves the load balancer IP.
GetLoadBalancerIP(ctx context.Context) (string, error)
// GetLoadBalancerEndpoint retrieves the load balancer endpoint.
GetLoadBalancerEndpoint(ctx context.Context) (string, error)
// GetInstance retrieves an instance using its providerID.
GetInstance(ctx context.Context, providerID string) (metadata.InstanceMetadata, error)
// Supported is used to determine if metadata API is implemented for this cloud provider.
@ -85,8 +85,8 @@ type ClusterAutoscaler interface {
}
type stubProviderMetadata struct {
GetLoadBalancerIPErr error
GetLoadBalancerIPResp string
GetLoadBalancerEndpointErr error
GetLoadBalancerEndpointResp string
GetSubnetworkCIDRErr error
GetSubnetworkCIDRResp string
@ -107,8 +107,8 @@ type stubProviderMetadata struct {
UIDResp string
}
func (m *stubProviderMetadata) GetLoadBalancerIP(ctx context.Context) (string, error) {
return m.GetLoadBalancerIPResp, m.GetLoadBalancerIPErr
func (m *stubProviderMetadata) GetLoadBalancerEndpoint(ctx context.Context) (string, error) {
return m.GetLoadBalancerEndpointResp, m.GetLoadBalancerEndpointErr
}
func (m *stubProviderMetadata) GetSubnetworkCIDR(ctx context.Context) (string, error) {

6
bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go
View file

@ -5,6 +5,7 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubelet"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/versions"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -17,7 +18,6 @@ import (
// Slimmed down to the fields we require
const (
bindPort = 6443
auditLogDir = "/var/log/kubernetes/audit/"
auditLogFile = "audit.log"
auditPolicyPath = "/etc/kubernetes/audit-policy.yaml"
@ -45,7 +45,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sV
},
// AdvertiseAddress will be overwritten later
LocalAPIEndpoint: kubeadm.APIEndpoint{
BindPort: bindPort,
BindPort: constants.KubernetesPort,
},
},
// https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3#ClusterConfiguration
@ -216,7 +216,7 @@ func (k *KubeadmJoinYAML) SetControlPlane(advertiseAddress string) {
k.JoinConfiguration.ControlPlane = &kubeadm.JoinControlPlane{
LocalAPIEndpoint: kubeadm.APIEndpoint{
AdvertiseAddress: advertiseAddress,
BindPort: 6443,
BindPort: constants.KubernetesPort,
},
}
k.JoinConfiguration.SkipPhases = []string{"control-plane-prepare/download-certs"}

81
bootstrapper/internal/kubernetes/kubernetes.go
View file

@ -3,10 +3,8 @@ package kubernetes
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"os/exec"
"strings"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi"
@ -15,6 +13,7 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/util"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/iproute"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/spf13/afero"
@ -93,7 +92,7 @@ func (k *KubeWrapper) InitCluster(
var publicIP string
var nodePodCIDR string
var subnetworkPodCIDR string
var controlPlaneEndpointIP string // this is the IP in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>" hence the unfortunate name
var controlPlaneEndpoint string // this is the endpoint in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>"
var nodeIP string
var validIPs []net.IP
@ -102,7 +101,7 @@ func (k *KubeWrapper) InitCluster(
log.Infof("Retrieving node metadata")
instance, err = k.providerMetadata.Self(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving own instance metadata failed: %w", err)
return nil, fmt.Errorf("retrieving own instance metadata: %w", err)
}
if instance.VPCIP != "" {
validIPs = append(validIPs, net.ParseIP(instance.VPCIP))
@ -120,18 +119,13 @@ func (k *KubeWrapper) InitCluster(
}
subnetworkPodCIDR, err = k.providerMetadata.GetSubnetworkCIDR(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving subnetwork CIDR failed: %w", err)
return nil, fmt.Errorf("retrieving subnetwork CIDR: %w", err)
}
controlPlaneEndpointIP = publicIP
controlPlaneEndpoint = publicIP
if k.providerMetadata.SupportsLoadBalancer() {
controlPlaneEndpointIP, err = k.providerMetadata.GetLoadBalancerIP(ctx)
controlPlaneEndpoint, err = k.providerMetadata.GetLoadBalancerEndpoint(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving load balancer IP failed: %w", err)
}
if k.cloudProvider == "gcp" {
if err := manuallySetLoadbalancerIP(ctx, controlPlaneEndpointIP); err != nil {
return nil, fmt.Errorf("setting load balancer IP failed: %w", err)
}
return nil, fmt.Errorf("retrieving load balancer endpoint: %w", err)
}
}
}
@ -139,7 +133,7 @@ func (k *KubeWrapper) InitCluster(
zap.String("nodeName", nodeName),
zap.String("providerID", providerID),
zap.String("nodeIP", nodeIP),
zap.String("controlPlaneEndpointIP", controlPlaneEndpointIP),
zap.String("controlPlaneEndpointEndpoint", controlPlaneEndpoint),
zap.String("podCIDR", subnetworkPodCIDR),
).Infof("Setting information for node")
@ -149,7 +143,7 @@ func (k *KubeWrapper) InitCluster(
initConfig.SetCertSANs([]string{publicIP, nodeIP})
initConfig.SetNodeName(nodeName)
initConfig.SetProviderID(providerID)
initConfig.SetControlPlaneEndpoint(controlPlaneEndpointIP)
initConfig.SetControlPlaneEndpoint(controlPlaneEndpoint)
initConfigYAML, err := initConfig.Marshal()
if err != nil {
return nil, fmt.Errorf("encoding kubeadm init configuration as YAML: %w", err)
@ -249,15 +243,20 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
}
nodeName := nodeInternalIP
var providerID string
var loadbalancerEndpoint string
if k.providerMetadata.Supported() {
log.Infof("Retrieving node metadata")
instance, err := k.providerMetadata.Self(ctx)
if err != nil {
return fmt.Errorf("retrieving own instance metadata failed: %w", err)
return fmt.Errorf("retrieving own instance metadata: %w", err)
}
providerID = instance.ProviderID
nodeName = instance.Name
nodeInternalIP = instance.VPCIP
loadbalancerEndpoint, err = k.providerMetadata.GetLoadBalancerEndpoint(ctx)
if err != nil {
return fmt.Errorf("retrieving loadbalancer endpoint: %w", err)
}
}
nodeName = k8sCompliantHostname(nodeName)
@ -267,8 +266,19 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
zap.String("nodeIP", nodeInternalIP),
).Infof("Setting information for node")
// Step 2: configure kubeadm join config
// Step 2: Remove load balancer from local routing table on GCP.
if k.cloudProvider == "gcp" {
ip, _, err := net.SplitHostPort(loadbalancerEndpoint)
if err != nil {
return fmt.Errorf("parsing load balancer IP: %w", err)
}
if err := iproute.RemoveFromLocalRoutingTable(ctx, ip); err != nil {
return fmt.Errorf("removing load balancer IP from routing table: %w", err)
}
log.Infof("Removed load balancer IP from routing table")
}
// Step 3: configure kubeadm join config
joinConfig := k.configProvider.JoinConfiguration(k.cloudControllerManager.Supported())
joinConfig.SetAPIServerEndpoint(args.APIServerEndpoint)
joinConfig.SetToken(args.Token)
@ -319,15 +329,15 @@ func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServ
}
ccmConfigMaps, err := k.cloudControllerManager.ConfigMaps(instance)
if err != nil {
return fmt.Errorf("defining ConfigMaps for CCM failed: %w", err)
return fmt.Errorf("defining ConfigMaps for CCM: %w", err)
}
ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance.ProviderID, cloudServiceAccountURI)
if err != nil {
return fmt.Errorf("defining Secrets for CCM failed: %w", err)
return fmt.Errorf("defining Secrets for CCM: %w", err)
}
ccmImage, err := k.cloudControllerManager.Image(k8sVersion)
if err != nil {
return fmt.Errorf("defining Image for CCM failed: %w", err)
return fmt.Errorf("defining Image for CCM: %w", err)
}
cloudControllerManagerConfiguration := resources.NewDefaultCloudControllerManagerDeployment(
@ -335,7 +345,7 @@ func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServ
k.cloudControllerManager.ExtraArgs(), k.cloudControllerManager.Volumes(), k.cloudControllerManager.VolumeMounts(), k.cloudControllerManager.Env(),
)
if err := k.clusterUtil.SetupCloudControllerManager(k.client, cloudControllerManagerConfiguration, ccmConfigMaps, ccmSecrets); err != nil {
return fmt.Errorf("failed to setup cloud-controller-manager: %w", err)
return fmt.Errorf("setting up cloud-controller-manager: %w", err)
}
return nil
@ -347,14 +357,14 @@ func (k *KubeWrapper) setupCloudNodeManager(k8sVersion versions.ValidK8sVersion)
}
nodeManagerImage, err := k.cloudNodeManager.Image(k8sVersion)
if err != nil {
return fmt.Errorf("defining Image for Node Manager failed: %w", err)
return fmt.Errorf("defining Image for Node Manager: %w", err)
}
cloudNodeManagerConfiguration := resources.NewDefaultCloudNodeManagerDeployment(
nodeManagerImage, k.cloudNodeManager.Path(), k.cloudNodeManager.ExtraArgs(),
)
if err := k.clusterUtil.SetupCloudNodeManager(k.client, cloudNodeManagerConfiguration); err != nil {
return fmt.Errorf("failed to setup cloud-node-manager: %w", err)
return fmt.Errorf("setting up cloud-node-manager: %w", err)
}
return nil
@ -366,13 +376,13 @@ func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata,
}
caSecrets, err := k.clusterAutoscaler.Secrets(instance.ProviderID, cloudServiceAccountURI)
if err != nil {
return fmt.Errorf("defining Secrets for cluster-autoscaler failed: %w", err)
return fmt.Errorf("defining Secrets for cluster-autoscaler: %w", err)
}
clusterAutoscalerConfiguration := resources.NewDefaultAutoscalerDeployment(k.clusterAutoscaler.Volumes(), k.clusterAutoscaler.VolumeMounts(), k.clusterAutoscaler.Env(), k8sVersion)
clusterAutoscalerConfiguration.SetAutoscalerCommand(k.clusterAutoscaler.Name(), autoscalingNodeGroups)
if err := k.clusterUtil.SetupAutoscaling(k.client, clusterAutoscalerConfiguration, caSecrets); err != nil {
return fmt.Errorf("failed to setup cluster-autoscaler: %w", err)
return fmt.Errorf("setting up cluster-autoscaler: %w", err)
}
return nil
@ -425,27 +435,6 @@ func (k *KubeWrapper) setupOperators(ctx context.Context) error {
return nil
}
// manuallySetLoadbalancerIP sets the loadbalancer IP of the first control plane during init.
// The GCP guest agent does this usually, but is deployed in the cluster that doesn't exist
// at this point. This is a workaround to set the loadbalancer IP manually, so kubeadm and kubelet
// can talk to the local Kubernetes API server using the loadbalancer IP.
func manuallySetLoadbalancerIP(ctx context.Context, ip string) error {
// https://github.com/GoogleCloudPlatform/guest-agent/blob/792fce795218633bcbde505fb3457a0b24f26d37/google_guest_agent/addresses.go#L179
if !strings.Contains(ip, "/") {
ip = ip + "/32"
}
args := []string{"route", "add", "to", "local", ip, "scope", "host", "dev", "ens3", "proto", "66"}
_, err := exec.CommandContext(ctx, "ip", args...).Output()
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return fmt.Errorf("ip route add (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
}
return fmt.Errorf("ip route add: %w", err)
}
return nil
}
// k8sCompliantHostname transforms a hostname to an RFC 1123 compliant, lowercase subdomain as required by Kubernetes node names.
// The following regex is used by k8s for validation: /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/ .
// Only a simple heuristic is used for now (to lowercase, replace underscores).

16
bootstrapper/internal/kubernetes/kubernetes_test.go
View file

@ -5,12 +5,14 @@ import (
"errors"
"net"
"regexp"
"strconv"
"testing"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/bootstrapper/role"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/stretchr/testify/assert"
@ -86,8 +88,8 @@ func TestInitCluster(t *testing.T) {
PublicIP: publicIP,
AliasIPRanges: []string{aliasIPRange},
},
GetLoadBalancerIPResp: loadbalancerIP,
SupportsLoadBalancerResp: true,
GetLoadBalancerEndpointResp: loadbalancerIP,
SupportsLoadBalancerResp: true,
},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{SupportedResp: false},
@ -148,9 +150,9 @@ func TestInitCluster(t *testing.T) {
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{
GetLoadBalancerIPErr: someErr,
SupportsLoadBalancerResp: true,
SupportedResp: true,
GetLoadBalancerEndpointErr: someErr,
SupportsLoadBalancerResp: true,
SupportedResp: true,
},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{},
@ -319,7 +321,7 @@ func TestInitCluster(t *testing.T) {
func TestJoinCluster(t *testing.T) {
someErr := errors.New("failed")
joinCommand := &kubeadm.BootstrapTokenDiscovery{
APIServerEndpoint: "192.0.2.0:6443",
APIServerEndpoint: "192.0.2.0:" + strconv.Itoa(constants.KubernetesPort),
Token: "kube-fake-token",
CACertHashes: []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"},
}
@ -419,7 +421,7 @@ func TestJoinCluster(t *testing.T) {
ControlPlane: &kubeadm.JoinControlPlane{
LocalAPIEndpoint: kubeadm.APIEndpoint{
AdvertiseAddress: "192.0.2.1",
BindPort: 6443,
BindPort: constants.KubernetesPort,
},
},
SkipPhases: []string{"control-plane-prepare/download-certs"},