Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-08-02 03:56:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

Bootstrapper

Browse source
This commit is contained in:
katexochen 2022-07-05 14:14:11 +02:00 • committed by Paul Meyer
parent 1af18e990d
commit 66b573ea5d
34 changed files with 492 additions and 202 deletions
Download patch file Download diff file Expand all files Collapse all files

2
bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go
View file

@ -88,7 +88,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube
},
},
},
CertSANs: []string{"127.0.0.1", "10.118.0.1"},
CertSANs: []string{"127.0.0.1"},
},
ControllerManager: kubeadm.ControlPlaneComponent{
ExtraArgs: map[string]string{

179
bootstrapper/internal/kubernetes/k8sapi/resources/gcp_guest_agent.go Normal file
View file

@ -0,0 +1,179 @@
package resources
import (
"github.com/edgelesssys/constellation/internal/secrets"
apps "k8s.io/api/apps/v1"
k8s "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type gcpGuestAgentDaemonset struct {
DaemonSet apps.DaemonSet
}
func NewGCPGuestAgentDaemonset() *gcpGuestAgentDaemonset {
return &gcpGuestAgentDaemonset{
DaemonSet: apps.DaemonSet{
TypeMeta: meta.TypeMeta{
APIVersion: "apps/v1",
Kind: "DaemonSet",
},
ObjectMeta: meta.ObjectMeta{
Name: "gcp-guest-agent",
Namespace: "kube-system",
Labels: map[string]string{
"k8s-app": "gcp-guest-agent",
"component": "gcp-guest-agent",
"kubernetes.io/cluster-service": "true",
},
},
Spec: apps.DaemonSetSpec{
Selector: &meta.LabelSelector{
MatchLabels: map[string]string{
"k8s-app": "gcp-guest-agent",
},
},
Template: k8s.PodTemplateSpec{
ObjectMeta: meta.ObjectMeta{
Labels: map[string]string{
"k8s-app": "gcp-guest-agent",
},
},
Spec: k8s.PodSpec{
PriorityClassName: "system-cluster-critical",
Tolerations: []k8s.Toleration{
{
Key: "node-role.kubernetes.io/master",
Operator: k8s.TolerationOpExists,
Effect: k8s.TaintEffectNoSchedule,
},
{
Key: "node-role.kubernetes.io/control-plane",
Operator: k8s.TolerationOpExists,
Effect: k8s.TaintEffectNoSchedule,
},
},
ImagePullSecrets: []k8s.LocalObjectReference{
{
Name: secrets.PullSecretName,
},
},
Containers: []k8s.Container{
{
Name: "gcp-guest-agent",
Image: gcpGuestImage,
SecurityContext: &k8s.SecurityContext{
Privileged: func(b bool) *bool { return &b }(true),
Capabilities: &k8s.Capabilities{
Add: []k8s.Capability{"NET_ADMIN"},
},
},
VolumeMounts: []k8s.VolumeMount{
{
Name: "etcssl",
ReadOnly: true,
MountPath: "/etc/ssl",
},
{
Name: "etcpki",
ReadOnly: true,
MountPath: "/etc/pki",
},
{
Name: "bin",
ReadOnly: true,
MountPath: "/bin",
},
{
Name: "usrbin",
ReadOnly: true,
MountPath: "/usr/bin",
},
{
Name: "usr",
ReadOnly: true,
MountPath: "/usr",
},
{
Name: "lib",
ReadOnly: true,
MountPath: "/lib",
},
{
Name: "lib64",
ReadOnly: true,
MountPath: "/lib64",
},
},
},
},
Volumes: []k8s.Volume{
{
Name: "etcssl",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{
Path: "/etc/ssl",
},
},
},
{
Name: "etcpki",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{
Path: "/etc/pki",
},
},
},
{
Name: "bin",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{
Path: "/bin",
},
},
},
{
Name: "usrbin",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{
Path: "/usr/bin",
},
},
},
{
Name: "usr",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{
Path: "/usr",
},
},
},
{
Name: "lib",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{
Path: "/lib",
},
},
},
{
Name: "lib64",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{
Path: "/lib64",
},
},
},
},
HostNetwork: true,
},
},
},
},
}
}
// Marshal marshals the access-manager deployment as YAML documents.
func (c *gcpGuestAgentDaemonset) Marshal() ([]byte, error) {
return MarshalK8SResources(c)
}

9
bootstrapper/internal/kubernetes/k8sapi/resources/images.go
View file

@ -2,10 +2,11 @@ package resources
const (
// Constellation images.
joinImage = "ghcr.io/edgelesssys/constellation/join-service:v1.2"
accessManagerImage = "ghcr.io/edgelesssys/constellation/access-manager:v1.2"
kmsImage = "ghcr.io/edgelesssys/constellation/kmsserver:v1.2"
verificationImage = "ghcr.io/edgelesssys/constellation/verification-service:v1.2"
joinImage = "ghcr.io/edgelesssys/constellation/join-service:feat-coordinator-selfactivation-node"
accessManagerImage = "ghcr.io/edgelesssys/constellation/access-manager:feat-coordinator-selfactivation-node"
kmsImage = "ghcr.io/edgelesssys/constellation/kmsserver:feat-coordinator-selfactivation-node"
verificationImage = "ghcr.io/edgelesssys/constellation/verification-service:feat-coordinator-selfactivation-node"
gcpGuestImage = "ghcr.io/edgelesssys/gcp-guest-agent:latest"
// external images.
clusterAutoscalerImage = "k8s.gcr.io/autoscaling/cluster-autoscaler:v1.23.0"

2
bootstrapper/internal/kubernetes/k8sapi/resources/joinservice.go
View file

@ -22,7 +22,7 @@ type joinServiceDaemonset struct {
}
// NewJoinServiceDaemonset returns a daemonset for the join service.
func NewJoinServiceDaemonset(csp, measurementsJSON, idJSON string) *joinServiceDaemonset {
func NewJoinServiceDaemonset(csp string, measurementsJSON, idJSON string) *joinServiceDaemonset {
return &joinServiceDaemonset{
ClusterRole: rbac.ClusterRole{
TypeMeta: meta.TypeMeta{

17
bootstrapper/internal/kubernetes/k8sapi/util.go
View file

@ -90,13 +90,13 @@ func (k *KubernetesUtil) InitCluster(ctx context.Context, initConfig []byte) err
if err != nil {
return fmt.Errorf("creating init config file %v: %w", initConfigFile.Name(), err)
}
defer os.Remove(initConfigFile.Name())
// defer os.Remove(initConfigFile.Name())
if _, err := initConfigFile.Write(initConfig); err != nil {
return fmt.Errorf("writing kubeadm init yaml config %v: %w", initConfigFile.Name(), err)
}
cmd := exec.CommandContext(ctx, kubeadmPath, "init", "--config", initConfigFile.Name())
cmd := exec.CommandContext(ctx, kubeadmPath, "init", "-v=5", "--config", initConfigFile.Name())
_, err = cmd.Output()
if err != nil {
var exitErr *exec.ExitError
@ -237,6 +237,11 @@ func (k *KubernetesUtil) SetupJoinService(kubectl Client, joinServiceConfigurati
return kubectl.Apply(joinServiceConfiguration, true)
}
// SetupGCPGuestAgent deploys the GCP guest agent daemon set.
func (k *KubernetesUtil) SetupGCPGuestAgent(kubectl Client, guestAgentDaemonset resources.Marshaler) error {
return kubectl.Apply(guestAgentDaemonset, true)
}
// SetupCloudControllerManager deploys the k8s cloud-controller-manager.
func (k *KubernetesUtil) SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error {
if err := kubectl.Apply(configMaps, true); err != nil {
@ -289,18 +294,18 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte) err
if err != nil {
return fmt.Errorf("creating join config file %v: %w", joinConfigFile.Name(), err)
}
defer os.Remove(joinConfigFile.Name())
// defer os.Remove(joinConfigFile.Name())
if _, err := joinConfigFile.Write(joinConfig); err != nil {
return fmt.Errorf("writing kubeadm init yaml config %v: %w", joinConfigFile.Name(), err)
}
// run `kubeadm join` to join a worker node to an existing Kubernetes cluster
cmd := exec.CommandContext(ctx, kubeadmPath, "join", "--config", joinConfigFile.Name())
cmd := exec.CommandContext(ctx, kubeadmPath, "join", "-v=5", "--config", joinConfigFile.Name())
if _, err := cmd.Output(); err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return fmt.Errorf("kubeadm join failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
return fmt.Errorf("kubeadm join failed (code %v) with: %s (full err: %s)", exitErr.ExitCode(), exitErr.Stderr, err)
}
return fmt.Errorf("kubeadm join: %w", err)
}
@ -334,7 +339,7 @@ func (k *KubernetesUtil) GetControlPlaneJoinCertificateKey(ctx context.Context)
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return "", fmt.Errorf("kubeadm upload-certs failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
return "", fmt.Errorf("kubeadm upload-certs failed (code %v) with: %s (full err: %s)", exitErr.ExitCode(), exitErr.Stderr, err)
}
return "", fmt.Errorf("kubeadm upload-certs: %w", err)
}

1
bootstrapper/internal/kubernetes/k8sutil.go
View file

@ -21,6 +21,7 @@ type clusterUtil interface {
SetupCloudNodeManager(kubectl k8sapi.Client, cloudNodeManagerConfiguration resources.Marshaler) error
SetupKMS(kubectl k8sapi.Client, kmsConfiguration resources.Marshaler) error
SetupVerificationService(kubectl k8sapi.Client, verificationServiceConfiguration resources.Marshaler) error
SetupGCPGuestAgent(kubectl k8sapi.Client, gcpGuestAgentConfiguration resources.Marshaler) error
StartKubelet() error
RestartKubelet() error
GetControlPlaneJoinCertificateKey(ctx context.Context) (string, error)

47
bootstrapper/internal/kubernetes/kubernetes.go
View file

@ -3,7 +3,9 @@ package kubernetes
import (
"context"
"encoding/json"
"errors"
"fmt"
"os/exec"
"strings"
"time"
@ -90,8 +92,7 @@ func (k *KubeWrapper) InitCluster(
var publicIP string
var nodePodCIDR string
var subnetworkPodCIDR string
// this is the IP in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>" hence the unfortunate name
var controlPlaneEndpointIP string
var controlPlaneEndpointIP string // this is the IP in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>" hence the unfortunate name
var nodeIP string
// Step 1: retrieve cloud metadata for Kubernetes configuration
@ -121,6 +122,11 @@ func (k *KubeWrapper) InitCluster(
if err != nil {
return nil, fmt.Errorf("retrieving load balancer IP failed: %w", err)
}
if k.cloudProvider == "gcp" {
if err := manuallySetLoadbalancerIP(ctx, controlPlaneEndpointIP); err != nil {
return nil, fmt.Errorf("setting load balancer IP failed: %w", err)
}
}
}
}
@ -188,6 +194,12 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("failed to setup verification service: %w", err)
}
if k.cloudProvider == "gcp" {
if err := k.clusterUtil.SetupGCPGuestAgent(k.client, resources.NewGCPGuestAgentDaemonset()); err != nil {
return nil, fmt.Errorf("failed to setup gcp guest agent: %w", err)
}
}
go k.clusterUtil.FixCilium(nodeName)
return k.GetKubeconfig()
@ -247,15 +259,7 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo
// GetKubeconfig returns the current nodes kubeconfig of stored on disk.
func (k *KubeWrapper) GetKubeconfig() ([]byte, error) {
kubeconf, err := k.kubeconfigReader.ReadKubeconfig()
if err != nil {
return nil, err
}
// replace the cluster.Server endpoint (127.0.0.1:16443) in admin.conf with the first bootstrapper endpoint (10.118.0.1:6443)
// kube-api server listens on 10.118.0.1:6443
// 127.0.0.1:16443 is the high availability balancer nginx endpoint, runnining localy on all nodes
// alternatively one could also start a local high availability balancer.
return []byte(strings.ReplaceAll(string(kubeconf), "127.0.0.1:16443", "10.118.0.1:6443")), nil
return k.kubeconfigReader.ReadKubeconfig()
}
// GetKubeadmCertificateKey return the key needed to join the Cluster as Control-Plane (has to be executed on a control-plane; errors otherwise).
@ -335,6 +339,27 @@ func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata,
return nil
}
// manuallySetLoadbalancerIP sets the loadbalancer IP of the first control plane during init.
// The GCP guest agent does this usually, but is deployed in the cluster that doesn't exist
// at this point. This is a workaround to set the loadbalancer IP manually, so kubeadm and kubelet
// can talk to the local Kubernetes API server using the loadbalancer IP.
func manuallySetLoadbalancerIP(ctx context.Context, ip string) error {
// https://github.com/GoogleCloudPlatform/guest-agent/blob/792fce795218633bcbde505fb3457a0b24f26d37/google_guest_agent/addresses.go#L179
if !strings.Contains(ip, "/") {
ip = ip + "/32"
}
args := fmt.Sprintf("route add to local %s scope host dev ens3 proto 66", ip)
_, err := exec.CommandContext(ctx, "ip", strings.Split(args, " ")...).Output()
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return fmt.Errorf("ip route add (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
}
return fmt.Errorf("ip route add: %w", err)
}
return nil
}
// k8sCompliantHostname transforms a hostname to an RFC 1123 compliant, lowercase subdomain as required by Kubernetes node names.
// The following regex is used by k8s for validation: /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/ .
// Only a simple heuristic is used for now (to lowercase, replace underscores).

33
bootstrapper/internal/kubernetes/kubernetes_test.go
View file

@ -441,34 +441,6 @@ func TestJoinCluster(t *testing.T) {
}
}
func TestGetKubeconfig(t *testing.T) {
testCases := map[string]struct {
Kubewrapper KubeWrapper
wantErr bool
}{
"check single replacement": {
Kubewrapper: KubeWrapper{kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("127.0.0.1:16443"),
}},
},
"check multiple replacement": {
Kubewrapper: KubeWrapper{kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("127.0.0.1:16443...127.0.0.1:16443"),
}},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
data, err := tc.Kubewrapper.GetKubeconfig()
require.NoError(err)
assert.NotContains(string(data), "127.0.0.1:16443")
assert.Contains(string(data), "10.118.0.1:6443")
})
}
}
func TestK8sCompliantHostname(t *testing.T) {
compliantHostname := regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`)
testCases := map[string]struct {
@ -512,6 +484,7 @@ type stubClusterUtil struct {
setupKMSError error
setupAccessManagerError error
setupVerificationServiceErr error
setupGCPGuestAgentErr error
joinClusterErr error
startKubeletErr error
restartKubeletErr error
@ -543,6 +516,10 @@ func (s *stubClusterUtil) SetupJoinService(kubectl k8sapi.Client, joinServiceCon
return s.setupJoinServiceError
}
func (s *stubClusterUtil) SetupGCPGuestAgent(kubectl k8sapi.Client, gcpGuestAgentConfiguration resources.Marshaler) error {
return s.setupGCPGuestAgentErr
}
func (s *stubClusterUtil) SetupCloudControllerManager(kubectl k8sapi.Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error {
return s.setupCloudControllerManagerError
}