Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-04-20 15:35:55 -04:00
Code Issues Packages Projects Releases Wiki Activity

kubernetes: set feature gate ControlPlaneKubeletLocalMode

Browse Source 
Co-Authored-By: Leonard Cohnen <lc@edgeless.systems>
This commit is contained in:
Markus Rudy 2024-10-10 20:04:08 +02:00
parent a73483f11a
commit 3e22aa8d43
4 changed files with 89 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel
View File

@ -28,6 +28,7 @@ go_library(
"@io_k8s_kubelet//config/v1beta1",
"@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3",
"@io_k8s_kubernetes//cmd/kubeadm/app/constants",
"@org_golang_x_mod//semver",
],
)

8
bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go
View File

@ -12,6 +12,7 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"golang.org/x/mod/semver"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletconf "k8s.io/kubelet/config/v1beta1"
@ -38,7 +39,7 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
cloudProvider = "external"
}
return KubeadmInitYAML{
initConfig := KubeadmInitYAML{
InitConfiguration: kubeadm.InitConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: kubeadm.SchemeGroupVersion.String(),
@ -157,6 +158,11 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
TLSPrivateKeyFile: certificate.KeyFilename,
},
}
if semver.Compare(clusterVersion, "v1.31.0") >= 0 {
initConfig.ClusterConfiguration.FeatureGates = map[string]bool{"ControlPlaneKubeletLocalMode": true}
}
return initConfig
}
// JoinConfiguration returns a new kubeadm join configuration.

132
internal/constellation/kubecmd/kubecmd.go
View File

@ -131,6 +131,18 @@ func (k *KubeCmd) UpgradeKubernetesVersion(ctx context.Context, kubernetesVersio
)
}
// TODO(burgerdev): remove after releasing v2.19
// Workaround for https://github.com/kubernetes/kubernetes/issues/127316: force kubelet to
// connect to the local API server.
if err := k.patchKubeadmConfig(ctx, func(cc *kubeadm.ClusterConfiguration) {
if cc.FeatureGates == nil {
cc.FeatureGates = map[string]bool{}
}
cc.FeatureGates["ControlPlaneKubeletLocalMode"] = true
}); err != nil {
return fmt.Errorf("setting FeatureGate ControlPlaneKubeletLocalMode: %w", err)
}
versionConfig, ok := versions.VersionConfigs[kubernetesVersion]
if !ok {
return fmt.Errorf("skipping Kubernetes upgrade: %w", compatibility.NewInvalidUpgradeError(
@ -236,65 +248,32 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At
// ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs.
// Empty strings are ignored, existing SANs are preserved.
func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error {
var kubeadmConfig *corev1.ConfigMap
if err := k.retryAction(ctx, func(ctx context.Context) error {
var err error
kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap)
return err
}); err != nil {
return fmt.Errorf("retrieving current kubeadm-config: %w", err)
}
clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey]
if !ok {
return errors.New("ClusterConfiguration missing from kubeadm-config")
}
var clusterConfiguration kubeadm.ClusterConfiguration
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil {
return fmt.Errorf("decoding cluster configuration data: %w", err)
}
existingSANs := make(map[string]struct{})
for _, existingSAN := range clusterConfiguration.APIServer.CertSANs {
existingSANs[existingSAN] = struct{}{}
}
var missingSANs []string
for _, san := range alternativeNames {
if san == "" {
continue // skip empty SANs
if err := k.patchKubeadmConfig(ctx, func(clusterConfiguration *kubeadm.ClusterConfiguration) {
existingSANs := make(map[string]struct{})
for _, existingSAN := range clusterConfiguration.APIServer.CertSANs {
existingSANs[existingSAN] = struct{}{}
}
if _, ok := existingSANs[san]; !ok {
missingSANs = append(missingSANs, san)
existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice
var missingSANs []string
for _, san := range alternativeNames {
if san == "" {
continue // skip empty SANs
}
if _, ok := existingSANs[san]; !ok {
missingSANs = append(missingSANs, san)
existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice
}
}
}
if len(missingSANs) == 0 {
k.log.Debug("No new SANs to add to the cluster's apiserver SAN field")
return nil
}
k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", "))
if len(missingSANs) == 0 {
k.log.Debug("No new SANs to add to the cluster's apiserver SAN field")
}
k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", "))
clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...)
sort.Strings(clusterConfiguration.APIServer.CertSANs)
opt := k8sjson.SerializerOptions{Yaml: true}
serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt)
encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion)
newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration)
if err != nil {
return fmt.Errorf("marshaling ClusterConfiguration: %w", err)
}
kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML)
k.log.Debug("Triggering kubeadm config update now")
if err = k.retryAction(ctx, func(ctx context.Context) error {
_, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig)
return err
clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...)
sort.Strings(clusterConfiguration.APIServer.CertSANs)
}); err != nil {
return fmt.Errorf("setting new kubeadm config: %w", err)
return fmt.Errorf("extending ClusterConfig.CertSANs: %w", err)
}
k.log.Debug("Successfully extended the cluster's apiserver SAN field")
@ -462,6 +441,51 @@ func (k *KubeCmd) retryAction(ctx context.Context, action func(ctx context.Conte
return retrier.Do(ctx)
}
// patchKubeadmConfig fetches and unpacks the kube-system/kubeadm-config ClusterConfiguration entry,
// runs doPatch on it and uploads the result.
func (k *KubeCmd) patchKubeadmConfig(ctx context.Context, doPatch func(*kubeadm.ClusterConfiguration)) error {
var kubeadmConfig *corev1.ConfigMap
if err := k.retryAction(ctx, func(ctx context.Context) error {
var err error
kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap)
return err
}); err != nil {
return fmt.Errorf("retrieving current kubeadm-config: %w", err)
}
clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey]
if !ok {
return errors.New("ClusterConfiguration missing from kubeadm-config")
}
var clusterConfiguration kubeadm.ClusterConfiguration
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil {
return fmt.Errorf("decoding cluster configuration data: %w", err)
}
doPatch(&clusterConfiguration)
opt := k8sjson.SerializerOptions{Yaml: true}
serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt)
encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion)
newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration)
if err != nil {
return fmt.Errorf("marshaling ClusterConfiguration: %w", err)
}
kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML)
k.log.Debug("Triggering kubeadm config update now")
if err = k.retryAction(ctx, func(ctx context.Context) error {
_, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig)
return err
}); err != nil {
return fmt.Errorf("setting new kubeadm config: %w", err)
}
k.log.Debug("Successfully patched the cluster's kubeadm-config")
return nil
}
func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error {
var err error
switch {

3
internal/constellation/kubecmd/kubecmd_test.go
View File

@ -281,6 +281,9 @@ func TestUpgradeKubernetesVersion(t *testing.T) {
}
kubectl := &stubKubectl{
unstructuredInterface: unstructuredClient,
configMaps: map[string]*corev1.ConfigMap{
constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": kubeadmClusterConfigurationV1Beta4}},
},
}
if tc.customClientFn != nil {
kubectl.unstructuredInterface = tc.customClientFn(nodeVersion)