kubernetes: set feature gate ControlPlaneKubeletLocalMode

Co-Authored-By: Leonard Cohnen <lc@edgeless.systems>

bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel

@@ -28,6 +28,7 @@ go_library(
         "@io_k8s_kubelet//config/v1beta1",
         "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3",
         "@io_k8s_kubernetes//cmd/kubeadm/app/constants",
+        "@org_golang_x_mod//semver",
     ],
 )
 

bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go

@@ -12,6 +12,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/edgelesssys/constellation/v2/internal/kubernetes"
+	"golang.org/x/mod/semver"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeletconf "k8s.io/kubelet/config/v1beta1"
@@ -38,7 +39,7 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
 		cloudProvider = "external"
 	}
 
-	return KubeadmInitYAML{
+	initConfig := KubeadmInitYAML{
 		InitConfiguration: kubeadm.InitConfiguration{
 			TypeMeta: metav1.TypeMeta{
 				APIVersion: kubeadm.SchemeGroupVersion.String(),
@@ -157,6 +158,11 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
 			TLSPrivateKeyFile: certificate.KeyFilename,
 		},
 	}
+
+	if semver.Compare(clusterVersion, "v1.31.0") >= 0 {
+		initConfig.ClusterConfiguration.FeatureGates = map[string]bool{"ControlPlaneKubeletLocalMode": true}
+	}
+	return initConfig
 }
 
 // JoinConfiguration returns a new kubeadm join configuration.

internal/constellation/kubecmd/kubecmd.go

@@ -131,6 +131,18 @@ func (k *KubeCmd) UpgradeKubernetesVersion(ctx context.Context, kubernetesVersio
 		)
 	}
 
+	// TODO(burgerdev): remove after releasing v2.19
+	// Workaround for https://github.com/kubernetes/kubernetes/issues/127316: force kubelet to
+	// connect to the local API server.
+	if err := k.patchKubeadmConfig(ctx, func(cc *kubeadm.ClusterConfiguration) {
+		if cc.FeatureGates == nil {
+			cc.FeatureGates = map[string]bool{}
+		}
+		cc.FeatureGates["ControlPlaneKubeletLocalMode"] = true
+	}); err != nil {
+		return fmt.Errorf("setting FeatureGate ControlPlaneKubeletLocalMode: %w", err)
+	}
+
 	versionConfig, ok := versions.VersionConfigs[kubernetesVersion]
 	if !ok {
 		return fmt.Errorf("skipping Kubernetes upgrade: %w", compatibility.NewInvalidUpgradeError(
@@ -236,25 +248,7 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At
 // ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs.
 // Empty strings are ignored, existing SANs are preserved.
 func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error {
-	var kubeadmConfig *corev1.ConfigMap
+	if err := k.patchKubeadmConfig(ctx, func(clusterConfiguration *kubeadm.ClusterConfiguration) {
-	if err := k.retryAction(ctx, func(ctx context.Context) error {
-		var err error
-		kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap)
-		return err
-	}); err != nil {
-		return fmt.Errorf("retrieving current kubeadm-config: %w", err)
-	}
-
-	clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey]
-	if !ok {
-		return errors.New("ClusterConfiguration missing from kubeadm-config")
-	}
-
-	var clusterConfiguration kubeadm.ClusterConfiguration
-	if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil {
-		return fmt.Errorf("decoding cluster configuration data: %w", err)
-	}
-
 		existingSANs := make(map[string]struct{})
 		for _, existingSAN := range clusterConfiguration.APIServer.CertSANs {
 			existingSANs[existingSAN] = struct{}{}
@@ -273,28 +267,13 @@ func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNa
 
 		if len(missingSANs) == 0 {
 			k.log.Debug("No new SANs to add to the cluster's apiserver SAN field")
-		return nil
 		}
 		k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", "))
 
 		clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...)
 		sort.Strings(clusterConfiguration.APIServer.CertSANs)
-
-	opt := k8sjson.SerializerOptions{Yaml: true}
-	serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt)
-	encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion)
-	newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration)
-	if err != nil {
-		return fmt.Errorf("marshaling ClusterConfiguration: %w", err)
-	}
-
-	kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML)
-	k.log.Debug("Triggering kubeadm config update now")
-	if err = k.retryAction(ctx, func(ctx context.Context) error {
-		_, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig)
-		return err
 	}); err != nil {
-		return fmt.Errorf("setting new kubeadm config: %w", err)
+		return fmt.Errorf("extending ClusterConfig.CertSANs: %w", err)
 	}
 
 	k.log.Debug("Successfully extended the cluster's apiserver SAN field")
@@ -462,6 +441,51 @@ func (k *KubeCmd) retryAction(ctx context.Context, action func(ctx context.Conte
 	return retrier.Do(ctx)
 }
 
+// patchKubeadmConfig fetches and unpacks the kube-system/kubeadm-config ClusterConfiguration entry,
+// runs doPatch on it and uploads the result.
+func (k *KubeCmd) patchKubeadmConfig(ctx context.Context, doPatch func(*kubeadm.ClusterConfiguration)) error {
+	var kubeadmConfig *corev1.ConfigMap
+	if err := k.retryAction(ctx, func(ctx context.Context) error {
+		var err error
+		kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap)
+		return err
+	}); err != nil {
+		return fmt.Errorf("retrieving current kubeadm-config: %w", err)
+	}
+
+	clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey]
+	if !ok {
+		return errors.New("ClusterConfiguration missing from kubeadm-config")
+	}
+
+	var clusterConfiguration kubeadm.ClusterConfiguration
+	if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil {
+		return fmt.Errorf("decoding cluster configuration data: %w", err)
+	}
+
+	doPatch(&clusterConfiguration)
+
+	opt := k8sjson.SerializerOptions{Yaml: true}
+	serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt)
+	encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion)
+	newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration)
+	if err != nil {
+		return fmt.Errorf("marshaling ClusterConfiguration: %w", err)
+	}
+
+	kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML)
+	k.log.Debug("Triggering kubeadm config update now")
+	if err = k.retryAction(ctx, func(ctx context.Context) error {
+		_, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig)
+		return err
+	}); err != nil {
+		return fmt.Errorf("setting new kubeadm config: %w", err)
+	}
+
+	k.log.Debug("Successfully patched the cluster's kubeadm-config")
+	return nil
+}
+
 func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error {
 	var err error
 	switch {

internal/constellation/kubecmd/kubecmd_test.go

@@ -281,6 +281,9 @@ func TestUpgradeKubernetesVersion(t *testing.T) {
 			}
 			kubectl := &stubKubectl{
 				unstructuredInterface: unstructuredClient,
+				configMaps: map[string]*corev1.ConfigMap{
+					constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": kubeadmClusterConfigurationV1Beta4}},
+				},
 			}
 			if tc.customClientFn != nil {
 				kubectl.unstructuredInterface = tc.customClientFn(nodeVersion)