constellation/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go

/*
Copyright (c) Edgeless Systems GmbH

SPDX-License-Identifier: AGPL-3.0-only
*/

package k8sapi

import (
	"path/filepath"

	"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubelet"
	"github.com/edgelesssys/constellation/v2/internal/constants"
	"github.com/edgelesssys/constellation/v2/internal/kubernetes"
	"github.com/edgelesssys/constellation/v2/internal/versions"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kubeletconf "k8s.io/kubelet/config/v1beta1"
	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
	kubeconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
)

// Uses types defined here: https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta3/
// Slimmed down to the fields we require

const (
	auditLogDir     = "/var/log/kubernetes/audit/"
	auditLogFile    = "audit.log"
	auditPolicyPath = "/etc/kubernetes/audit-policy.yaml"
)

// KubdeadmConfiguration is used to generate kubeadm configurations.
type KubdeadmConfiguration struct{}

// InitConfiguration returns a new init configuration.
func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, k8sVersion versions.ValidK8sVersion) KubeadmInitYAML {
	var cloudProvider string
	if externalCloudProvider {
		cloudProvider = "external"
	}

	return KubeadmInitYAML{
		InitConfiguration: kubeadm.InitConfiguration{
			TypeMeta: metav1.TypeMeta{
				APIVersion: kubeadm.SchemeGroupVersion.String(),
				Kind:       "InitConfiguration",
			},
			NodeRegistration: kubeadm.NodeRegistrationOptions{
				CRISocket: "unix:///run/containerd/containerd.sock",
				KubeletExtraArgs: map[string]string{
					"cloud-provider": cloudProvider,
				},
			},
			// AdvertiseAddress will be overwritten later
			LocalAPIEndpoint: kubeadm.APIEndpoint{
				BindPort: constants.KubernetesPort,
			},
		},
		// https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3#ClusterConfiguration
		ClusterConfiguration: kubeadm.ClusterConfiguration{
			TypeMeta: metav1.TypeMeta{
				Kind:       "ClusterConfiguration",
				APIVersion: kubeadm.SchemeGroupVersion.String(),
			},
			// Target kubernetes version of the control plane.
			KubernetesVersion: versions.VersionConfigs[k8sVersion].PatchVersion,
			// necessary to be able to access the kubeapi server through localhost
			APIServer: kubeadm.APIServer{
				ControlPlaneComponent: kubeadm.ControlPlaneComponent{
					ExtraArgs: map[string]string{
						"audit-policy-file":           auditPolicyPath,
						"audit-log-path":              filepath.Join(auditLogDir, auditLogFile), // CIS benchmark
						"audit-log-maxage":            "30",                                     // CIS benchmark - Default value of Rancher
						"audit-log-maxbackup":         "10",                                     // CIS benchmark - Default value of Rancher
						"audit-log-maxsize":           "100",                                    // CIS benchmark - Default value of Rancher
						"profiling":                   "false",                                  // CIS benchmark
						"egress-selector-config-file": "/etc/kubernetes/egress-selector-configuration.yaml",
						"kubelet-certificate-authority": filepath.Join(
							kubeconstants.KubernetesDir,
							kubeconstants.DefaultCertificateDir,
							kubeconstants.CACertName,
						),
						"tls-cipher-suites": "TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256," +
							"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256," +
							"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384," +
							"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256," +
							"TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," +
							"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305," +
							"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA," +
							"TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384", // CIS benchmark
					},
					ExtraVolumes: []kubeadm.HostPathMount{
						{
							Name:      "audit-log",
							HostPath:  auditLogDir,
							MountPath: auditLogDir,
							ReadOnly:  false,
							PathType:  corev1.HostPathDirectoryOrCreate,
						},
						{
							Name:      "audit",
							HostPath:  auditPolicyPath,
							MountPath: auditPolicyPath,
							ReadOnly:  true,
							PathType:  corev1.HostPathFile,
						},
						{
							Name:      "egress-config",
							HostPath:  "/etc/kubernetes/egress-selector-configuration.yaml",
							MountPath: "/etc/kubernetes/egress-selector-configuration.yaml",
							ReadOnly:  true,
							PathType:  corev1.HostPathFile,
						},
						{
							Name:      "konnectivity-uds",
							HostPath:  "/etc/kubernetes/konnectivity-server",
							MountPath: "/etc/kubernetes/konnectivity-server",
							ReadOnly:  false,
							PathType:  corev1.HostPathDirectoryOrCreate,
						},
					},
				},
				CertSANs: []string{"127.0.0.1"},
			},
			ControllerManager: kubeadm.ControlPlaneComponent{
				ExtraArgs: map[string]string{
					"flex-volume-plugin-dir":      "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/",
					"cloud-provider":              cloudProvider,
					"configure-cloud-routes":      "false",
					"profiling":                   "false", // CIS benchmark
					"terminated-pod-gc-threshold": "1000",  // CIS benchmark - Default value of Rancher
				},
			},
			Scheduler: kubeadm.ControlPlaneComponent{
				ExtraArgs: map[string]string{
					"profiling": "false",
				},
			},
		},
		// warning: this config is applied to every node in the cluster!
		KubeletConfiguration: kubeletconf.KubeletConfiguration{
			ProtectKernelDefaults: true, // CIS benchmark
			TLSCipherSuites: []string{
				"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
				"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
				"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
				"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
				"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
				"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
				"TLS_RSA_WITH_AES_256_GCM_SHA384",
				"TLS_RSA_WITH_AES_128_GCM_SHA256",
			}, // CIS benchmark
			StaticPodPath: "/etc/kubernetes/manifests",
			TypeMeta: metav1.TypeMeta{
				APIVersion: kubeletconf.SchemeGroupVersion.String(),
				Kind:       "KubeletConfiguration",
			},
			RegisterWithTaints: []corev1.Taint{
				{
					Key:    "node.cloudprovider.kubernetes.io/uninitialized",
					Value:  "true",
					Effect: corev1.TaintEffectPreferNoSchedule,
				},
				{
					Key:    "node.cilium.io/agent-not-ready",
					Value:  "true",
					Effect: corev1.TaintEffectPreferNoSchedule,
				},
			},
			TLSCertFile:       kubelet.CertificateFilename,
			TLSPrivateKeyFile: kubelet.KeyFilename,
		},
	}
}

// JoinConfiguration returns a new kubeadm join configuration.
func (c *KubdeadmConfiguration) JoinConfiguration(externalCloudProvider bool) KubeadmJoinYAML {
	var cloudProvider string
	if externalCloudProvider {
		cloudProvider = "external"
	}
	return KubeadmJoinYAML{
		JoinConfiguration: kubeadm.JoinConfiguration{
			TypeMeta: metav1.TypeMeta{
				APIVersion: kubeadm.SchemeGroupVersion.String(),
				Kind:       "JoinConfiguration",
			},
			NodeRegistration: kubeadm.NodeRegistrationOptions{
				CRISocket: "unix:///run/containerd/containerd.sock",
				KubeletExtraArgs: map[string]string{
					"cloud-provider": cloudProvider,
				},
			},
			Discovery: kubeadm.Discovery{
				BootstrapToken: &kubeadm.BootstrapTokenDiscovery{},
			},
		},
		KubeletConfiguration: kubeletconf.KubeletConfiguration{
			TypeMeta: metav1.TypeMeta{
				APIVersion: kubeletconf.SchemeGroupVersion.String(),
				Kind:       "KubeletConfiguration",
			},
		},
	}
}

// KubeadmJoinYAML holds configuration for kubeadm join workflow.
type KubeadmJoinYAML struct {
	JoinConfiguration    kubeadm.JoinConfiguration
	KubeletConfiguration kubeletconf.KubeletConfiguration
}

// SetNodeName sets the node name.
func (k *KubeadmJoinYAML) SetNodeName(nodeName string) {
	k.JoinConfiguration.NodeRegistration.Name = nodeName
}

// SetAPIServerEndpoint sets the api server endpoint.
func (k *KubeadmJoinYAML) SetAPIServerEndpoint(apiServerEndpoint string) {
	k.JoinConfiguration.Discovery.BootstrapToken.APIServerEndpoint = apiServerEndpoint
}

// SetToken sets the boostrap token.
func (k *KubeadmJoinYAML) SetToken(token string) {
	k.JoinConfiguration.Discovery.BootstrapToken.Token = token
}

// AppendDiscoveryTokenCaCertHash appends another trusted discovery token CA hash.
func (k *KubeadmJoinYAML) AppendDiscoveryTokenCaCertHash(discoveryTokenCaCertHash string) {
	k.JoinConfiguration.Discovery.BootstrapToken.CACertHashes = append(k.JoinConfiguration.Discovery.BootstrapToken.CACertHashes, discoveryTokenCaCertHash)
}

// SetNodeIP sets the node IP.
func (k *KubeadmJoinYAML) SetNodeIP(nodeIP string) {
	if k.JoinConfiguration.NodeRegistration.KubeletExtraArgs == nil {
		k.JoinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{"node-ip": nodeIP}
	} else {
		k.JoinConfiguration.NodeRegistration.KubeletExtraArgs["node-ip"] = nodeIP
	}
}

// SetProviderID sets the provider ID.
func (k *KubeadmJoinYAML) SetProviderID(providerID string) {
	k.KubeletConfiguration.ProviderID = providerID
}

// SetControlPlane sets the control plane with the advertised address.
func (k *KubeadmJoinYAML) SetControlPlane(advertiseAddress string) {
	k.JoinConfiguration.ControlPlane = &kubeadm.JoinControlPlane{
		LocalAPIEndpoint: kubeadm.APIEndpoint{
			AdvertiseAddress: advertiseAddress,
			BindPort:         constants.KubernetesPort,
		},
	}
	k.JoinConfiguration.SkipPhases = []string{"control-plane-prepare/download-certs"}
}

// Marshal into a k8s resource YAML.
func (k *KubeadmJoinYAML) Marshal() ([]byte, error) {
	return kubernetes.MarshalK8SResources(k)
}

// Unmarshal from a k8s resource YAML.
func (k *KubeadmJoinYAML) Unmarshal(yamlData []byte) (KubeadmJoinYAML, error) {
	var tmp KubeadmJoinYAML
	return tmp, kubernetes.UnmarshalK8SResources(yamlData, &tmp)
}

// KubeadmInitYAML holds configuration for kubeadm init workflow.
type KubeadmInitYAML struct {
	InitConfiguration    kubeadm.InitConfiguration
	ClusterConfiguration kubeadm.ClusterConfiguration
	KubeletConfiguration kubeletconf.KubeletConfiguration
}

// SetNodeName sets name of node.
func (k *KubeadmInitYAML) SetNodeName(nodeName string) {
	k.InitConfiguration.NodeRegistration.Name = nodeName
}

// SetCertSANs sets the SANs for the certificate.
func (k *KubeadmInitYAML) SetCertSANs(certSANs []string) {
	for _, certSAN := range certSANs {
		if certSAN == "" {
			continue
		}
		k.ClusterConfiguration.APIServer.CertSANs = append(k.ClusterConfiguration.APIServer.CertSANs, certSAN)
	}
}

// SetAPIServerAdvertiseAddress sets the advertised API server address.
func (k *KubeadmInitYAML) SetAPIServerAdvertiseAddress(apiServerAdvertiseAddress string) {
	k.InitConfiguration.LocalAPIEndpoint.AdvertiseAddress = apiServerAdvertiseAddress
}

// SetControlPlaneEndpoint sets the control plane endpoint if controlPlaneEndpoint is not empty.
func (k *KubeadmInitYAML) SetControlPlaneEndpoint(controlPlaneEndpoint string) {
	if controlPlaneEndpoint != "" {
		k.ClusterConfiguration.ControlPlaneEndpoint = controlPlaneEndpoint
	}
}

// SetServiceCIDR sets the CIDR of service subnet.
func (k *KubeadmInitYAML) SetServiceCIDR(serviceCIDR string) {
	k.ClusterConfiguration.Networking.ServiceSubnet = serviceCIDR
}

// SetPodNetworkCIDR sets the CIDR of pod subnet.
func (k *KubeadmInitYAML) SetPodNetworkCIDR(podNetworkCIDR string) {
	k.ClusterConfiguration.Networking.PodSubnet = podNetworkCIDR
}

// SetServiceDNSDomain sets the dns domain.
func (k *KubeadmInitYAML) SetServiceDNSDomain(serviceDNSDomain string) {
	k.ClusterConfiguration.Networking.DNSDomain = serviceDNSDomain
}

// SetNodeIP sets the node IP.
func (k *KubeadmInitYAML) SetNodeIP(nodeIP string) {
	if k.InitConfiguration.NodeRegistration.KubeletExtraArgs == nil {
		k.InitConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{"node-ip": nodeIP}
	} else {
		k.InitConfiguration.NodeRegistration.KubeletExtraArgs["node-ip"] = nodeIP
	}
}

// SetProviderID sets the provider ID.
func (k *KubeadmInitYAML) SetProviderID(providerID string) {
	if k.InitConfiguration.NodeRegistration.KubeletExtraArgs == nil {
		k.InitConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{"provider-id": providerID}
	} else {
		k.InitConfiguration.NodeRegistration.KubeletExtraArgs["provider-id"] = providerID
	}
}

// Marshal into a k8s resource YAML.
func (k *KubeadmInitYAML) Marshal() ([]byte, error) {
	return kubernetes.MarshalK8SResources(k)
}

// Unmarshal from a k8s resource YAML.
func (k *KubeadmInitYAML) Unmarshal(yamlData []byte) (KubeadmInitYAML, error) {
	var tmp KubeadmInitYAML
	return tmp, kubernetes.UnmarshalK8SResources(yamlData, &tmp)
}