AB#2074: Choosable K8S Version (#277)

AB#2074: Add configurable k8s version

Configurable version flow:
* cli config holds/validates k8sVersion
* InitCluster receive a k8sVersion arg
* InitCluster creates CM "k8s-version"
* kubeadm's InitConfiguration receives k8sVersion
* joinservice spec mounts/reads k8s-version CM
* joinservice supplies k8sVersion via JoinTicketResponse
Other changes:
* Remove unused test code (FakeK8SClient)
* move VersionConfig map to /internal/versions
* installk8sComponents is now a function instead of a method

bootstrapper/cmd/bootstrapper/test.go

@@ -21,7 +21,7 @@ func (c *clusterFake) InitCluster(context.Context, []string, string, string, att
 }
 
 // JoinCluster will fake joining the current node to an existing cluster.
-func (c *clusterFake) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, *logger.Logger) error {
+func (c *clusterFake) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, string, *logger.Logger) error {
 	return nil
 }
 

bootstrapper/internal/joinclient/client.go

@@ -272,7 +272,7 @@ func (c *JoinClient) startNodeAndJoin(ticket *joinproto.IssueJoinTicketResponse,
 		Token:             ticket.Token,
 		CACertHashes:      []string{ticket.DiscoveryTokenCaCertHash},
 	}
-	if err := c.joiner.JoinCluster(ctx, btd, c.role, c.log); err != nil {
+	if err := c.joiner.JoinCluster(ctx, btd, c.role, ticket.KubernetesVersion, c.log); err != nil {
 		return fmt.Errorf("joining Kubernetes cluster: %w", err)
 	}
 
@@ -386,6 +386,7 @@ type ClusterJoiner interface {
 		ctx context.Context,
 		args *kubeadm.BootstrapTokenDiscovery,
 		peerRole role.Role,
+		k8sVersion string,
 		log *logger.Logger,
 	) error
 }

bootstrapper/internal/joinclient/client_test.go

@@ -385,7 +385,7 @@ type stubClusterJoiner struct {
 	joinClusterErr    error
 }
 
-func (j *stubClusterJoiner) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, *logger.Logger) error {
+func (j *stubClusterJoiner) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, string, *logger.Logger) error {
 	j.joinClusterCalled = true
 	return j.joinClusterErr
 }

bootstrapper/internal/kubernetes/k8sapi/constants.go

@@ -0,0 +1,16 @@
+package k8sapi
+
+const (
+	// Paths and permissions necessary for Kubernetes installation.
+	cniPluginsDir           = "/opt/cni/bin"
+	binDir                  = "/run/state/bin"
+	kubeadmPath             = "/run/state/bin/kubeadm"
+	kubeletPath             = "/run/state/bin/kubelet"
+	kubectlPath             = "/run/state/bin/kubectl"
+	kubeletServiceEtcPath   = "/etc/systemd/system/kubelet.service"
+	kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service"
+	kubeadmConfEtcPath      = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
+	kubeadmConfStatePath    = "/run/state/systemd/system/kubelet.service.d/10-kubeadm.conf"
+	executablePerm          = 0o544
+	systemdUnitPerm         = 0o644
+)

bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/edgelesssys/constellation/bootstrapper/internal/kubelet"
 	"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
-	"github.com/edgelesssys/constellation/internal/constants"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeletconf "k8s.io/kubelet/config/v1beta1"
@@ -25,7 +24,7 @@ const (
 
 type CoreOSConfiguration struct{}
 
-func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) KubeadmInitYAML {
+func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sVersion string) KubeadmInitYAML {
 	var cloudProvider string
 	if externalCloudProvider {
 		cloudProvider = "external"
@@ -48,12 +47,14 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube
 				BindPort: bindPort,
 			},
 		},
+		// https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3#ClusterConfiguration
 		ClusterConfiguration: kubeadm.ClusterConfiguration{
 			TypeMeta: metav1.TypeMeta{
 				Kind:       "ClusterConfiguration",
 				APIVersion: kubeadm.SchemeGroupVersion.String(),
 			},
-			KubernetesVersion: constants.KubernetesVersion,
+			// Target kubernetes version of the control plane.
+			KubernetesVersion: k8sVersion,
 			// necessary to be able to access the kubeapi server through localhost
 			APIServer: kubeadm.APIServer{
 				ControlPlaneComponent: kubeadm.ControlPlaneComponent{

bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go

@@ -19,11 +19,11 @@ func TestInitConfiguration(t *testing.T) {
 		config KubeadmInitYAML
 	}{
 		"CoreOS init config can be created": {
-			config: coreOSConfig.InitConfiguration(true),
+			config: coreOSConfig.InitConfiguration(true, "3.2.1"),
 		},
 		"CoreOS init config with all fields can be created": {
 			config: func() KubeadmInitYAML {
-				c := coreOSConfig.InitConfiguration(true)
+				c := coreOSConfig.InitConfiguration(true, "3.2.1")
 				c.SetAPIServerAdvertiseAddress("192.0.2.0")
 				c.SetNodeIP("192.0.2.0")
 				c.SetNodeName("node")

bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go

@@ -2,9 +2,11 @@ package client
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 
 	"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -88,3 +90,13 @@ func (c *Client) GetObjects(resources resources.Marshaler) ([]*resource.Info, er
 		Do()
 	return result.Infos()
 }
+
+// CreateConfigMap creates the given ConfigMap.
+func (c *Client) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
+	_, err := c.clientset.CoreV1().ConfigMaps(configMap.ObjectMeta.Namespace).Create(ctx, &configMap, metav1.CreateOptions{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go

@@ -1,10 +1,12 @@
 package kubectl
 
 import (
+	"context"
 	"errors"
 	"fmt"
 
 	"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/cli-runtime/pkg/resource"
 )
 
@@ -17,6 +19,7 @@ type Client interface {
 	ApplyOneObject(info *resource.Info, forceConflicts bool) error
 	// GetObjects converts resources into prepared info fields for use in ApplyOneObject.
 	GetObjects(resources resources.Marshaler) ([]*resource.Info, error)
+	CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
 }
 
 // clientGenerator can generate new clients from a kubeconfig.
@@ -66,3 +69,17 @@ func (k *Kubectl) Apply(resources resources.Marshaler, forceConflicts bool) erro
 func (k *Kubectl) SetKubeconfig(kubeconfig []byte) {
 	k.kubeconfig = kubeconfig
 }
+
+func (k *Kubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
+	client, err := k.clientGenerator.NewClient(k.kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	err = client.CreateConfigMap(ctx, configMap)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go

@@ -1,12 +1,14 @@
 package kubectl
 
 import (
+	"context"
 	"errors"
 	"testing"
 
 	"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
 	"github.com/stretchr/testify/assert"
 	"go.uber.org/goleak"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/cli-runtime/pkg/resource"
 )
 
@@ -15,9 +17,10 @@ func TestMain(m *testing.M) {
 }
 
 type stubClient struct {
-	applyOneObjectErr error
+	applyOneObjectErr  error
-	getObjectsInfos   []*resource.Info
+	getObjectsInfos    []*resource.Info
-	getObjectsErr     error
+	getObjectsErr      error
+	createConfigMapErr error
 }
 
 func (s *stubClient) ApplyOneObject(info *resource.Info, forceConflicts bool) error {
@@ -28,11 +31,16 @@ func (s *stubClient) GetObjects(resources resources.Marshaler) ([]*resource.Info
 	return s.getObjectsInfos, s.getObjectsErr
 }
 
+func (s *stubClient) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
+	return s.createConfigMapErr
+}
+
 type stubClientGenerator struct {
-	applyOneObjectErr error
+	applyOneObjectErr  error
-	getObjectsInfos   []*resource.Info
+	getObjectsInfos    []*resource.Info
-	getObjectsErr     error
+	getObjectsErr      error
-	newClientErr      error
+	newClientErr       error
+	createConfigMapErr error
 }
 
 func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) {
@@ -40,6 +48,7 @@ func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) {
 		s.applyOneObjectErr,
 		s.getObjectsInfos,
 		s.getObjectsErr,
+		s.createConfigMapErr,
 	}, s.newClientErr
 }
 

bootstrapper/internal/kubernetes/k8sapi/resources/access_manager.go

@@ -2,6 +2,7 @@ package resources
 
 import (
 	"github.com/edgelesssys/constellation/internal/secrets"
+	"github.com/edgelesssys/constellation/internal/versions"
 	"google.golang.org/protobuf/proto"
 	apps "k8s.io/api/apps/v1"
 	k8s "k8s.io/api/core/v1"
@@ -104,7 +105,7 @@ func NewAccessManagerDeployment(sshUsers map[string]string) *accessManagerDeploy
 						InitContainers: []k8s.Container{
 							{
 								Name:  "constellation-access-manager",
-								Image: accessManagerImage,
+								Image: versions.AccessManagerImage,
 								VolumeMounts: []k8s.VolumeMount{
 									{
 										Name:      "host",

bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go

@@ -1,6 +1,7 @@
 package resources
 
 import (
+	"github.com/edgelesssys/constellation/internal/versions"
 	"google.golang.org/protobuf/proto"
 	apps "k8s.io/api/apps/v1"
 	k8s "k8s.io/api/core/v1"
@@ -434,7 +435,7 @@ func NewDefaultAutoscalerDeployment(extraVolumes []k8s.Volume, extraVolumeMounts
 						Containers: []k8s.Container{
 							{
 								Name:            "cluster-autoscaler",
-								Image:           clusterAutoscalerImage,
+								Image:           versions.ClusterAutoscalerImage,
 								ImagePullPolicy: k8s.PullIfNotPresent,
 								LivenessProbe: &k8s.Probe{
 									ProbeHandler: k8s.ProbeHandler{

bootstrapper/internal/kubernetes/k8sapi/resources/gcp_guest_agent.go

@@ -2,6 +2,7 @@ package resources
 
 import (
 	"github.com/edgelesssys/constellation/internal/secrets"
+	"github.com/edgelesssys/constellation/internal/versions"
 	apps "k8s.io/api/apps/v1"
 	k8s "k8s.io/api/core/v1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -64,7 +65,7 @@ func NewGCPGuestAgentDaemonset() *gcpGuestAgentDaemonset {
 						Containers: []k8s.Container{
 							{
 								Name:  "gcp-guest-agent",
-								Image: gcpGuestImage, // built from https://github.com/edgelesssys/gcp-guest-agent
+								Image: versions.GcpGuestImage, // built from https://github.com/edgelesssys/gcp-guest-agent
 								SecurityContext: &k8s.SecurityContext{
 									Privileged: func(b bool) *bool { return &b }(true),
 									Capabilities: &k8s.Capabilities{

bootstrapper/internal/kubernetes/k8sapi/resources/images.go

@@ -1,13 +0,0 @@
-package resources
-
-const (
-	// Constellation images.
-	joinImage          = "ghcr.io/edgelesssys/constellation/join-service:v1.3.2-0.20220714151638-d295be31"
-	accessManagerImage = "ghcr.io/edgelesssys/constellation/access-manager:v1.3.2-0.20220714151638-d295be31"
-	kmsImage           = "ghcr.io/edgelesssys/constellation/kmsserver:v1.3.2-0.20220714151638-d295be31"
-	verificationImage  = "ghcr.io/edgelesssys/constellation/verification-service:v1.3.2-0.20220714151638-d295be31"
-	gcpGuestImage      = "ghcr.io/edgelesssys/gcp-guest-agent:latest"
-
-	// external images.
-	clusterAutoscalerImage = "k8s.gcr.io/autoscaling/cluster-autoscaler:v1.23.0"
-)

bootstrapper/internal/kubernetes/k8sapi/resources/joinservice.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/edgelesssys/constellation/internal/constants"
 	"github.com/edgelesssys/constellation/internal/secrets"
+	"github.com/edgelesssys/constellation/internal/versions"
 	apps "k8s.io/api/apps/v1"
 	k8s "k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
@@ -135,7 +136,7 @@ func NewJoinServiceDaemonset(csp string, measurementsJSON, idJSON string) *joinS
 						Containers: []k8s.Container{
 							{
 								Name:  "join-service",
-								Image: joinImage,
+								Image: versions.JoinImage,
 								Ports: []k8s.ContainerPort{
 									{
 										ContainerPort: constants.JoinServicePort,
@@ -167,9 +168,22 @@ func NewJoinServiceDaemonset(csp string, measurementsJSON, idJSON string) *joinS
 							{
 								Name: "config",
 								VolumeSource: k8s.VolumeSource{
-									ConfigMap: &k8s.ConfigMapVolumeSource{
+									Projected: &k8s.ProjectedVolumeSource{
-										LocalObjectReference: k8s.LocalObjectReference{
+										Sources: []k8s.VolumeProjection{
-											Name: "join-config",
+											{
+												ConfigMap: &k8s.ConfigMapProjection{
+													LocalObjectReference: k8s.LocalObjectReference{
+														Name: "join-config",
+													},
+												},
+											},
+											{
+												ConfigMap: &k8s.ConfigMapProjection{
+													LocalObjectReference: k8s.LocalObjectReference{
+														Name: constants.K8sVersion,
+													},
+												},
+											},
 										},
 									},
 								},

bootstrapper/internal/kubernetes/k8sapi/resources/kms.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/edgelesssys/constellation/internal/constants"
 	"github.com/edgelesssys/constellation/internal/secrets"
+	"github.com/edgelesssys/constellation/internal/versions"
 	apps "k8s.io/api/apps/v1"
 	k8s "k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
@@ -226,7 +227,7 @@ func NewKMSDeployment(csp string, masterSecret []byte) *kmsDeployment {
 						Containers: []k8s.Container{
 							{
 								Name:  "kms",
-								Image: kmsImage,
+								Image: versions.KmsImage,
 								Args: []string{
 									fmt.Sprintf("--atls-port=%d", constants.KMSATLSPort),
 									fmt.Sprintf("--port=%d", constants.KMSPort),

bootstrapper/internal/kubernetes/k8sapi/resources/verification.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/edgelesssys/constellation/internal/constants"
 	"github.com/edgelesssys/constellation/internal/secrets"
+	"github.com/edgelesssys/constellation/internal/versions"
 	apps "k8s.io/api/apps/v1"
 	k8s "k8s.io/api/core/v1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -73,7 +74,7 @@ func NewVerificationDaemonSet(csp string) *verificationDaemonset {
 						Containers: []k8s.Container{
 							{
 								Name:  "verification-service",
-								Image: verificationImage,
+								Image: versions.VerificationImage,
 								Ports: []k8s.ContainerPort{
 									{
 										Name:          "http",

bootstrapper/internal/kubernetes/k8sapi/util.go

@@ -7,6 +7,7 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
+	"io/fs"
 	"net"
 	"net/http"
 	"os"
@@ -21,8 +22,12 @@ import (
 	"github.com/edgelesssys/constellation/bootstrapper/util"
 	"github.com/edgelesssys/constellation/internal/file"
 	"github.com/edgelesssys/constellation/internal/logger"
+	"github.com/edgelesssys/constellation/internal/versions"
+	"github.com/icholy/replace"
 	"github.com/spf13/afero"
 	"go.uber.org/zap"
+	"golang.org/x/text/transform"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
@@ -39,9 +44,17 @@ var providerIDRegex = regexp.MustCompile(`^azure:///subscriptions/([^/]+)/resour
 type Client interface {
 	Apply(resources resources.Marshaler, forceConflicts bool) error
 	SetKubeconfig(kubeconfig []byte)
+	CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
 	// TODO: add tolerations
 }
 
+type installer interface {
+	Install(
+		ctx context.Context, sourceURL string, destinations []string, perm fs.FileMode,
+		extract bool, transforms ...transform.Transformer,
+	) error
+}
+
 // KubernetesUtil provides low level management of the kubernetes cluster.
 type KubernetesUtil struct {
 	inst installer
@@ -58,13 +71,46 @@ func NewKubernetesUtil() *KubernetesUtil {
 
 // InstallComponents installs kubernetes components in the version specified.
 func (k *KubernetesUtil) InstallComponents(ctx context.Context, version string) error {
-	var versionConf kubernetesVersion
+	var versionConf versions.KubernetesVersion
 	var ok bool
-	if versionConf, ok = versionConfigs[version]; !ok {
+	if versionConf, ok = versions.VersionConfigs[version]; !ok {
 		return fmt.Errorf("unsupported kubernetes version %q", version)
 	}
-	if err := versionConf.installK8sComponents(ctx, k.inst); err != nil {
+
-		return err
+	if err := k.inst.Install(
+		ctx, versionConf.CNIPluginsURL, []string{cniPluginsDir}, executablePerm, true,
+	); err != nil {
+		return fmt.Errorf("installing cni plugins: %w", err)
+	}
+	if err := k.inst.Install(
+		ctx, versionConf.CrictlURL, []string{binDir}, executablePerm, true,
+	); err != nil {
+		return fmt.Errorf("installing crictl: %w", err)
+	}
+	if err := k.inst.Install(
+		ctx, versionConf.KubeletServiceURL, []string{kubeletServiceEtcPath, kubeletServiceStatePath}, systemdUnitPerm, false, replace.String("/usr/bin", binDir),
+	); err != nil {
+		return fmt.Errorf("installing kubelet service: %w", err)
+	}
+	if err := k.inst.Install(
+		ctx, versionConf.KubeadmConfURL, []string{kubeadmConfEtcPath, kubeadmConfStatePath}, systemdUnitPerm, false, replace.String("/usr/bin", binDir),
+	); err != nil {
+		return fmt.Errorf("installing kubeadm conf: %w", err)
+	}
+	if err := k.inst.Install(
+		ctx, versionConf.KubeletURL, []string{kubeletPath}, executablePerm, false,
+	); err != nil {
+		return fmt.Errorf("installing kubelet: %w", err)
+	}
+	if err := k.inst.Install(
+		ctx, versionConf.KubeadmURL, []string{kubeadmPath}, executablePerm, false,
+	); err != nil {
+		return fmt.Errorf("installing kubeadm: %w", err)
+	}
+	if err := k.inst.Install(
+		ctx, versionConf.KubectlURL, []string{kubectlPath}, executablePerm, false,
+	); err != nil {
+		return fmt.Errorf("installing kubectl: %w", err)
 	}
 
 	return enableSystemdUnit(ctx, kubeletServiceEtcPath)

bootstrapper/internal/kubernetes/k8sapi/versions.go

@@ -1,95 +0,0 @@
-package k8sapi
-
-import (
-	"context"
-	"fmt"
-	"io/fs"
-
-	"github.com/icholy/replace"
-	"golang.org/x/text/transform"
-)
-
-const (
-	cniPluginsDir           = "/opt/cni/bin"
-	binDir                  = "/run/state/bin"
-	kubeadmPath             = "/run/state/bin/kubeadm"
-	kubeletPath             = "/run/state/bin/kubelet"
-	kubectlPath             = "/run/state/bin/kubectl"
-	kubeletServiceEtcPath   = "/etc/systemd/system/kubelet.service"
-	kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service"
-	kubeadmConfEtcPath      = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
-	kubeadmConfStatePath    = "/run/state/systemd/system/kubelet.service.d/10-kubeadm.conf"
-	executablePerm          = 0o544
-	systemdUnitPerm         = 0o644
-)
-
-// versionConfigs holds download URLs for all required kubernetes components for every supported version.
-var versionConfigs map[string]kubernetesVersion = map[string]kubernetesVersion{
-	"1.23.6": {
-		CNIPluginsURL:     "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz",
-		CrictlURL:         "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.1/crictl-v1.24.1-linux-amd64.tar.gz",
-		KubeletServiceURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service",
-		KubeadmConfURL:    "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf",
-		KubeletURL:        "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubelet",
-		KubeadmURL:        "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubeadm",
-		KubectlURL:        "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl",
-	},
-}
-
-type kubernetesVersion struct {
-	CNIPluginsURL     string
-	CrictlURL         string
-	KubeletServiceURL string
-	KubeadmConfURL    string
-	KubeletURL        string
-	KubeadmURL        string
-	KubectlURL        string
-}
-
-// installK8sComponents installs kubernetes components for this version.
-// reference: https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#installing-kubeadm-kubelet-and-kubectl .
-func (k *kubernetesVersion) installK8sComponents(ctx context.Context, inst installer) error {
-	if err := inst.Install(
-		ctx, k.CNIPluginsURL, []string{cniPluginsDir}, executablePerm, true,
-	); err != nil {
-		return fmt.Errorf("installing cni plugins: %w", err)
-	}
-	if err := inst.Install(
-		ctx, k.CrictlURL, []string{binDir}, executablePerm, true,
-	); err != nil {
-		return fmt.Errorf("installing crictl: %w", err)
-	}
-	if err := inst.Install(
-		ctx, k.KubeletServiceURL, []string{kubeletServiceEtcPath, kubeletServiceStatePath}, systemdUnitPerm, false, replace.String("/usr/bin", binDir),
-	); err != nil {
-		return fmt.Errorf("installing kubelet service: %w", err)
-	}
-	if err := inst.Install(
-		ctx, k.KubeadmConfURL, []string{kubeadmConfEtcPath, kubeadmConfStatePath}, systemdUnitPerm, false, replace.String("/usr/bin", binDir),
-	); err != nil {
-		return fmt.Errorf("installing kubeadm conf: %w", err)
-	}
-	if err := inst.Install(
-		ctx, k.KubeletURL, []string{kubeletPath}, executablePerm, false,
-	); err != nil {
-		return fmt.Errorf("installing kubelet: %w", err)
-	}
-	if err := inst.Install(
-		ctx, k.KubeadmURL, []string{kubeadmPath}, executablePerm, false,
-	); err != nil {
-		return fmt.Errorf("installing kubeadm: %w", err)
-	}
-	if err := inst.Install(
-		ctx, k.KubectlURL, []string{kubectlPath}, executablePerm, false,
-	); err != nil {
-		return fmt.Errorf("installing kubectl: %w", err)
-	}
-	return nil
-}
-
-type installer interface {
-	Install(
-		ctx context.Context, sourceURL string, destinations []string, perm fs.FileMode,
-		extract bool, transforms ...transform.Transformer,
-	) error
-}

bootstrapper/internal/kubernetes/kubernetes.go

@@ -16,8 +16,11 @@ import (
 	attestationtypes "github.com/edgelesssys/constellation/internal/attestation/types"
 	"github.com/edgelesssys/constellation/internal/cloud/metadata"
 	"github.com/edgelesssys/constellation/internal/logger"
+	"github.com/edgelesssys/constellation/internal/versions"
 	"github.com/spf13/afero"
 	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
 )
 
@@ -28,7 +31,7 @@ type configReader interface {
 
 // configurationProvider provides kubeadm init and join configuration.
 type configurationProvider interface {
-	InitConfiguration(externalCloudProvider bool) k8sapi.KubeadmInitYAML
+	InitConfiguration(externalCloudProvider bool, k8sVersion string) k8sapi.KubeadmInitYAML
 	JoinConfiguration(externalCloudProvider bool) k8sapi.KubeadmJoinYAML
 }
 
@@ -79,7 +82,6 @@ func (k *KubeWrapper) InitCluster(
 	ctx context.Context, autoscalingNodeGroups []string, cloudServiceAccountURI, k8sVersion string,
 	id attestationtypes.ID, kmsConfig KMSConfig, sshUsers map[string]string, log *logger.Logger,
 ) ([]byte, error) {
-	// TODO: k8s version should be user input
 	log.With(zap.String("version", k8sVersion)).Infof("Installing Kubernetes components")
 	if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil {
 		return nil, err
@@ -149,7 +151,7 @@ func (k *KubeWrapper) InitCluster(
 	).Infof("Setting information for node")
 
 	// Step 2: configure kubeadm init config
-	initConfig := k.configProvider.InitConfiguration(k.cloudControllerManager.Supported())
+	initConfig := k.configProvider.InitConfiguration(k.cloudControllerManager.Supported(), k8sVersion)
 	initConfig.SetNodeIP(nodeIP)
 	initConfig.SetCertSANs([]string{publicIP, nodeIP})
 	initConfig.SetNodeName(nodeName)
@@ -219,16 +221,21 @@ func (k *KubeWrapper) InitCluster(
 		}
 	}
 
+	// Store the received k8sVersion in a ConfigMap, overwriting exisiting values (there shouldn't be any).
+	// Joining nodes determine the kubernetes version they will install based on this ConfigMap.
+	if err := k.setupK8sVersionConfigMap(ctx, k8sVersion, true); err != nil {
+		return nil, fmt.Errorf("failed to setup k8s version ConfigMap: %v", err)
+	}
+
 	k.clusterUtil.FixCilium(nodeName)
 
 	return k.GetKubeconfig()
 }
 
 // JoinCluster joins existing Kubernetes cluster.
-func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, log *logger.Logger) error {
+func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, k8sVersion string, log *logger.Logger) error {
-	// TODO: k8s version should be user input
+	log.With(zap.String("version", k8sVersion)).Infof("Installing Kubernetes components")
-	log.With(zap.String("version", "1.23.6")).Infof("Installing Kubernetes components")
+	if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil {
-	if err := k.clusterUtil.InstallComponents(ctx, "1.23.6"); err != nil {
 		return err
 	}
 
@@ -357,6 +364,35 @@ func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata,
 	return nil
 }
 
+// setupK8sVersionConfigMap applies a ConfigMap (cf. server-side apply) to consistently store the installed k8s version.
+func (k *KubeWrapper) setupK8sVersionConfigMap(ctx context.Context, k8sVersion string, forceConflicts bool) error {
+	if !versions.IsSupportedK8sVersion(k8sVersion) {
+		return fmt.Errorf("supplied k8s version is not supported: %v", k8sVersion)
+	}
+
+	config := corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "k8s-version",
+			Namespace: "kube-system",
+		},
+		Data: map[string]string{
+			"K8sVersion": k8sVersion,
+		},
+	}
+
+	// We do not use the client's Apply method here since we are handling a kubernetes-native type.
+	// These types don't implement our custom Marshaler interface.
+	if err := k.client.CreateConfigMap(ctx, config); err != nil {
+		return fmt.Errorf("Apply in KubeWrapper.setupK8sVersionConfigMap(..) failed with: %v", err)
+	}
+
+	return nil
+}
+
 // manuallySetLoadbalancerIP sets the loadbalancer IP of the first control plane during init.
 // The GCP guest agent does this usually, but is deployed in the cluster that doesn't exist
 // at this point. This is a workaround to set the loadbalancer IP manually, so kubeadm and kubelet
@@ -391,22 +427,3 @@ func k8sCompliantHostname(in string) string {
 func (k *KubeWrapper) StartKubelet() error {
 	return k.clusterUtil.StartKubelet()
 }
-
-type fakeK8SClient struct {
-	kubeconfig []byte
-}
-
-// NewFakeK8SClient creates a new, fake k8s client where apply always works.
-func NewFakeK8SClient([]byte) (k8sapi.Client, error) {
-	return &fakeK8SClient{}, nil
-}
-
-// Apply fakes applying Kubernetes resources.
-func (f *fakeK8SClient) Apply(resources resources.Marshaler, forceConflicts bool) error {
-	return nil
-}
-
-// SetKubeconfig stores the kubeconfig given to it.
-func (f *fakeK8SClient) SetKubeconfig(kubeconfig []byte) {
-	f.kubeconfig = kubeconfig
-}

bootstrapper/internal/kubernetes/kubernetes_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
+	corev1 "k8s.io/api/core/v1"
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
 )
 
@@ -35,11 +36,10 @@ func TestInitCluster(t *testing.T) {
 	publicIP := "192.0.2.2"
 	loadbalancerIP := "192.0.2.3"
 	aliasIPRange := "192.0.2.0/24"
-	k8sVersion := "1.23.8"
 
 	testCases := map[string]struct {
 		clusterUtil            stubClusterUtil
-		kubeCTL                stubKubeCTL
+		kubectl                stubKubectl
 		providerMetadata       ProviderMetadata
 		CloudControllerManager CloudControllerManager
 		CloudNodeManager       CloudNodeManager
@@ -47,6 +47,7 @@ func TestInitCluster(t *testing.T) {
 		kubeconfigReader       configReader
 		wantConfig             k8sapi.KubeadmInitYAML
 		wantErr                bool
+		k8sVersion             string
 	}{
 		"kubeadm init works without metadata": {
 			clusterUtil: stubClusterUtil{},
@@ -69,6 +70,7 @@ func TestInitCluster(t *testing.T) {
 				},
 				ClusterConfiguration: kubeadm.ClusterConfiguration{},
 			},
+			k8sVersion: "1.23.6",
 		},
 		"kubeadm init works with metadata and loadbalancer": {
 			clusterUtil: stubClusterUtil{},
@@ -107,7 +109,8 @@ func TestInitCluster(t *testing.T) {
 					},
 				},
 			},
-			wantErr: false,
+			wantErr:    false,
+			k8sVersion: "1.23.6",
 		},
 		"kubeadm init fails when retrieving metadata self": {
 			clusterUtil: stubClusterUtil{},
@@ -122,6 +125,7 @@ func TestInitCluster(t *testing.T) {
 			CloudNodeManager:       &stubCloudNodeManager{},
 			ClusterAutoscaler:      &stubClusterAutoscaler{},
 			wantErr:                true,
+			k8sVersion:             "1.23.6",
 		},
 		"kubeadm init fails when retrieving metadata subnetwork cidr": {
 			clusterUtil: stubClusterUtil{},
@@ -136,6 +140,7 @@ func TestInitCluster(t *testing.T) {
 			CloudNodeManager:       &stubCloudNodeManager{},
 			ClusterAutoscaler:      &stubClusterAutoscaler{},
 			wantErr:                true,
+			k8sVersion:             "1.23.6",
 		},
 		"kubeadm init fails when retrieving metadata loadbalancer ip": {
 			clusterUtil: stubClusterUtil{},
@@ -151,6 +156,7 @@ func TestInitCluster(t *testing.T) {
 			CloudNodeManager:       &stubCloudNodeManager{},
 			ClusterAutoscaler:      &stubClusterAutoscaler{},
 			wantErr:                true,
+			k8sVersion:             "1.23.6",
 		},
 		"kubeadm init fails when applying the init config": {
 			clusterUtil: stubClusterUtil{initClusterErr: someErr},
@@ -162,6 +168,7 @@ func TestInitCluster(t *testing.T) {
 			CloudNodeManager:       &stubCloudNodeManager{},
 			ClusterAutoscaler:      &stubClusterAutoscaler{},
 			wantErr:                true,
+			k8sVersion:             "1.23.6",
 		},
 		"kubeadm init fails when setting up the pod network": {
 			clusterUtil: stubClusterUtil{setupPodNetworkErr: someErr},
@@ -173,6 +180,7 @@ func TestInitCluster(t *testing.T) {
 			CloudNodeManager:       &stubCloudNodeManager{},
 			ClusterAutoscaler:      &stubClusterAutoscaler{},
 			wantErr:                true,
+			k8sVersion:             "1.23.6",
 		},
 		"kubeadm init fails when setting up the join service": {
 			clusterUtil: stubClusterUtil{setupJoinServiceError: someErr},
@@ -184,6 +192,7 @@ func TestInitCluster(t *testing.T) {
 			CloudNodeManager:       &stubCloudNodeManager{},
 			ClusterAutoscaler:      &stubClusterAutoscaler{},
 			wantErr:                true,
+			k8sVersion:             "1.23.6",
 		},
 		"kubeadm init fails when setting the cloud contoller manager": {
 			clusterUtil: stubClusterUtil{setupCloudControllerManagerError: someErr},
@@ -195,6 +204,7 @@ func TestInitCluster(t *testing.T) {
 			CloudNodeManager:       &stubCloudNodeManager{},
 			ClusterAutoscaler:      &stubClusterAutoscaler{},
 			wantErr:                true,
+			k8sVersion:             "1.23.6",
 		},
 		"kubeadm init fails when setting the cloud node manager": {
 			clusterUtil: stubClusterUtil{setupCloudNodeManagerError: someErr},
@@ -206,6 +216,7 @@ func TestInitCluster(t *testing.T) {
 			CloudNodeManager:       &stubCloudNodeManager{SupportedResp: true},
 			ClusterAutoscaler:      &stubClusterAutoscaler{},
 			wantErr:                true,
+			k8sVersion:             "1.23.6",
 		},
 		"kubeadm init fails when setting the cluster autoscaler": {
 			clusterUtil: stubClusterUtil{setupAutoscalingError: someErr},
@@ -217,6 +228,7 @@ func TestInitCluster(t *testing.T) {
 			CloudNodeManager:       &stubCloudNodeManager{},
 			ClusterAutoscaler:      &stubClusterAutoscaler{SupportedResp: true},
 			wantErr:                true,
+			k8sVersion:             "1.23.6",
 		},
 		"kubeadm init fails when reading kubeconfig": {
 			clusterUtil: stubClusterUtil{},
@@ -228,6 +240,7 @@ func TestInitCluster(t *testing.T) {
 			CloudNodeManager:       &stubCloudNodeManager{},
 			ClusterAutoscaler:      &stubClusterAutoscaler{},
 			wantErr:                true,
+			k8sVersion:             "1.23.6",
 		},
 		"kubeadm init fails when setting up the kms": {
 			clusterUtil: stubClusterUtil{setupKMSError: someErr},
@@ -239,6 +252,7 @@ func TestInitCluster(t *testing.T) {
 			CloudNodeManager:       &stubCloudNodeManager{SupportedResp: false},
 			ClusterAutoscaler:      &stubClusterAutoscaler{},
 			wantErr:                true,
+			k8sVersion:             "1.23.6",
 		},
 		"kubeadm init fails when setting up verification service": {
 			clusterUtil: stubClusterUtil{setupVerificationServiceErr: someErr},
@@ -250,6 +264,19 @@ func TestInitCluster(t *testing.T) {
 			CloudNodeManager:       &stubCloudNodeManager{SupportedResp: false},
 			ClusterAutoscaler:      &stubClusterAutoscaler{},
 			wantErr:                true,
+			k8sVersion:             "1.23.6",
+		},
+		"unsupported k8sVersion fails cluster creation": {
+			clusterUtil: stubClusterUtil{},
+			kubeconfigReader: &stubKubeconfigReader{
+				Kubeconfig: []byte("someKubeconfig"),
+			},
+			providerMetadata:       &stubProviderMetadata{},
+			CloudControllerManager: &stubCloudControllerManager{},
+			CloudNodeManager:       &stubCloudNodeManager{},
+			ClusterAutoscaler:      &stubClusterAutoscaler{},
+			k8sVersion:             "invalid version",
+			wantErr:                true,
 		},
 	}
 
@@ -265,11 +292,11 @@ func TestInitCluster(t *testing.T) {
 				cloudNodeManager:       tc.CloudNodeManager,
 				clusterAutoscaler:      tc.ClusterAutoscaler,
 				configProvider:         &stubConfigProvider{InitConfig: k8sapi.KubeadmInitYAML{}},
-				client:                 &tc.kubeCTL,
+				client:                 &tc.kubectl,
 				kubeconfigReader:       tc.kubeconfigReader,
 				getIPAddr:              func() (string, error) { return privateIP, nil },
 			}
-			_, err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountURI, k8sVersion, attestationtypes.ID{}, KMSConfig{MasterSecret: masterSecret}, nil, logger.NewTest(t))
+			_, err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountURI, tc.k8sVersion, attestationtypes.ID{}, KMSConfig{MasterSecret: masterSecret}, nil, logger.NewTest(t))
 
 			if tc.wantErr {
 				assert.Error(err)
@@ -294,6 +321,8 @@ func TestJoinCluster(t *testing.T) {
 	}
 
 	privateIP := "192.0.2.1"
+	// stubClusterUtil does not validate the k8sVersion, thus it can be arbitrary.
+	k8sVersion := "3.2.1"
 
 	testCases := map[string]struct {
 		clusterUtil            stubClusterUtil
@@ -425,7 +454,7 @@ func TestJoinCluster(t *testing.T) {
 				getIPAddr:              func() (string, error) { return privateIP, nil },
 			}
 
-			err := kube.JoinCluster(context.Background(), joinCommand, tc.role, logger.NewTest(t))
+			err := kube.JoinCluster(context.Background(), joinCommand, tc.role, k8sVersion, logger.NewTest(t))
 			if tc.wantErr {
 				assert.Error(err)
 				return
@@ -559,7 +588,7 @@ type stubConfigProvider struct {
 	JoinConfig k8sapi.KubeadmJoinYAML
 }
 
-func (s *stubConfigProvider) InitConfiguration(_ bool) k8sapi.KubeadmInitYAML {
+func (s *stubConfigProvider) InitConfiguration(_ bool, _ string) k8sapi.KubeadmInitYAML {
 	return s.InitConfig
 }
 
@@ -574,22 +603,27 @@ func (s *stubConfigProvider) JoinConfiguration(_ bool) k8sapi.KubeadmJoinYAML {
 	return s.JoinConfig
 }
 
-type stubKubeCTL struct {
+type stubKubectl struct {
-	ApplyErr error
+	ApplyErr           error
+	createConfigMapErr error
 
 	resources   []resources.Marshaler
 	kubeconfigs [][]byte
 }
 
-func (s *stubKubeCTL) Apply(resources resources.Marshaler, forceConflicts bool) error {
+func (s *stubKubectl) Apply(resources resources.Marshaler, forceConflicts bool) error {
 	s.resources = append(s.resources, resources)
 	return s.ApplyErr
 }
 
-func (s *stubKubeCTL) SetKubeconfig(kubeconfig []byte) {
+func (s *stubKubectl) SetKubeconfig(kubeconfig []byte) {
 	s.kubeconfigs = append(s.kubeconfigs, kubeconfig)
 }
 
+func (s *stubKubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
+	return s.createConfigMapErr
+}
+
 type stubKubeconfigReader struct {
 	Kubeconfig []byte
 	ReadErr    error

cli/internal/cmd/init.go

@@ -120,7 +120,7 @@ func initialize(cmd *cobra.Command, dialer grpcDialer, serviceAccCreator service
 		KeyEncryptionKeyId:     "",
 		UseExistingKek:         false,
 		CloudServiceAccountUri: serviceAccount,
-		KubernetesVersion:      "1.23.6",
+		KubernetesVersion:      config.KubernetesVersion,
 		SshUserKeys:            ssh.ToProtoSlice(sshUsers),
 	}
 	resp, err := initCall(cmd.Context(), dialer, stat.BootstrapperHost, req)

hack/go.mod

@@ -56,6 +56,7 @@ require (
 	github.com/emirpasic/gods v1.12.0 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.3.1 // indirect
+	github.com/icholy/replace v0.5.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect

hack/go.sum

@@ -636,6 +636,8 @@ github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63
 github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/icholy/replace v0.5.0 h1:Nx80zYQVlowdba+3Y6dvHDnmxaGtBrDlf2wYn9GyIXQ=
+github.com/icholy/replace v0.5.0/go.mod h1:zzi8pxElj2t/5wHHHYmH45D+KxytX/t4w3ClY5nlK+g=
 github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
@@ -1344,6 +1346,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -1654,6 +1657,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 h1:dbuHpmKjkDzSOMKAWl10QNlgaZUd3V1q99xc81tt2Kc=
 gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/config/config.go

@@ -11,6 +11,7 @@ import (
 	"github.com/edgelesssys/constellation/internal/cloud/cloudprovider"
 	"github.com/edgelesssys/constellation/internal/constants"
 	"github.com/edgelesssys/constellation/internal/file"
+	"github.com/edgelesssys/constellation/internal/versions"
 	"github.com/go-playground/locales/en"
 	ut "github.com/go-playground/universal-translator"
 	"github.com/go-playground/validator/v10"
@@ -60,6 +61,9 @@ type Config struct {
 	// examples:
 	//   - value: '[]UserKey{ { Username:  "Alice", PublicKey: "ssh-rsa AAAAB3NzaC...5QXHKW1rufgtJeSeJ8= alice@domain.com" } }'
 	SSHUsers []UserKey `yaml:"sshUsers,omitempty" validate:"dive"`
+	// description: |
+	//   Kubernetes version installed in the cluster.
+	KubernetesVersion string `yaml:"kubernetesVersion" validate:"supported_k8s_version"`
 }
 
 // UserKey describes a user that should be created with corresponding public SSH key.
@@ -226,9 +230,14 @@ func Default() *Config {
 				Measurements: qemuPCRs,
 			},
 		},
+		KubernetesVersion: "1.23.6",
 	}
 }
 
+func validateK8sVersion(fl validator.FieldLevel) bool {
+	return versions.IsSupportedK8sVersion(fl.Field().String())
+}
+
 // Validate checks the config values and returns validation error messages.
 // The function only returns an error if the validation itself fails.
 func (c *Config) Validate() ([]string, error) {
@@ -238,6 +247,11 @@ func (c *Config) Validate() ([]string, error) {
 		return nil, err
 	}
 
+	// register custom validator with label supported_k8s_version to validate version based on available versionConfigs.
+	if err := validate.RegisterValidation("supported_k8s_version", validateK8sVersion); err != nil {
+		return nil, err
+	}
+
 	err := validate.Struct(c)
 	if err == nil {
 		return nil, nil

internal/constants/constants.go

@@ -63,12 +63,14 @@ const (
 	// Filenames for Constellation's micro services.
 	//
 
-	// ServiceBasePath is the base path for the mounted micro services files.
+	// ServiceBasePath is the base path for the mounted micro service's files.
 	ServiceBasePath = "/var/config"
 	// MeasurementsFilename is the filename of CC measurements.
 	MeasurementsFilename = "measurements"
 	// IDFilename is the filename of Constellation's IDs.
 	IDFilename = "id"
+	// K8sVersion is the filename of the mapped k8s-version configMap file.
+	K8sVersion = "k8s-version"
 
 	//
 	// Cryptographic constants.

internal/versions/containerImages.go

@@ -0,0 +1,13 @@
+package versions
+
+const (
+	// Constellation images.
+	JoinImage          = "ghcr.io/edgelesssys/constellation/join-service:v1.3.2-0.20220714151638-d295be31"
+	AccessManagerImage = "ghcr.io/edgelesssys/constellation/access-manager:v1.3.2-0.20220714151638-d295be31"
+	KmsImage           = "ghcr.io/edgelesssys/constellation/kmsserver:v1.3.2-0.20220714151638-d295be31"
+	VerificationImage  = "ghcr.io/edgelesssys/constellation/verification-service:v1.3.2-0.20220714151638-d295be31"
+	GcpGuestImage      = "ghcr.io/edgelesssys/gcp-guest-agent:latest"
+
+	// external images.
+	ClusterAutoscalerImage = "k8s.gcr.io/autoscaling/cluster-autoscaler:v1.23.0"
+)

internal/versions/k8sBinaries.go

@@ -0,0 +1,31 @@
+package versions
+
+// versionConfigs holds download URLs for all required kubernetes components for every supported version.
+var VersionConfigs map[string]KubernetesVersion = map[string]KubernetesVersion{
+	"1.23.6": {
+		CNIPluginsURL:     "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz",
+		CrictlURL:         "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.1/crictl-v1.24.1-linux-amd64.tar.gz",
+		KubeletServiceURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service",
+		KubeadmConfURL:    "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf",
+		KubeletURL:        "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubelet",
+		KubeadmURL:        "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubeadm",
+		KubectlURL:        "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl",
+	},
+}
+
+type KubernetesVersion struct {
+	CNIPluginsURL     string
+	CrictlURL         string
+	KubeletServiceURL string
+	KubeadmConfURL    string
+	KubeletURL        string
+	KubeadmURL        string
+	KubectlURL        string
+}
+
+func IsSupportedK8sVersion(version string) bool {
+	if _, ok := VersionConfigs[version]; !ok {
+		return false
+	}
+	return true
+}

joinservice/internal/server/server.go

@@ -12,6 +12,7 @@ import (
 	"github.com/edgelesssys/constellation/internal/file"
 	"github.com/edgelesssys/constellation/internal/grpc/grpclog"
 	"github.com/edgelesssys/constellation/internal/logger"
+	"github.com/edgelesssys/constellation/internal/versions"
 	"github.com/edgelesssys/constellation/joinservice/joinproto"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
@@ -92,6 +93,12 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi
 		return nil, status.Errorf(codes.Internal, "unable to generate Kubernetes join arguments: %s", err)
 	}
 
+	log.Infof("Querying K8sVersion ConfigMap")
+	k8sVersion, err := s.getK8sVersion(ctx)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "unable to get k8s version: %s", err)
+	}
+
 	log.Infof("Creating signed kubelet certificate")
 	kubeletCert, err := s.ca.GetCertificate(req.CertificateRequest)
 	if err != nil {
@@ -125,9 +132,25 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi
 		DiscoveryTokenCaCertHash: kubeArgs.CACertHashes[0],
 		KubeletCert:              kubeletCert,
 		ControlPlaneFiles:        controlPlaneFiles,
+		KubernetesVersion:        k8sVersion,
 	}, nil
 }
 
+// getK8sVersion reads the k8s version from a VolumeMount that is backed by the k8s-version ConfigMap.
+func (s *Server) getK8sVersion(ctx context.Context) (string, error) {
+	fileContent, err := s.file.Read(filepath.Join(constants.ServiceBasePath, "k8s-version"))
+	if err != nil {
+		return "", fmt.Errorf("could not read k8s version file: %v", err)
+	}
+	k8sVersion := string(fileContent)
+
+	if !versions.IsSupportedK8sVersion(k8sVersion) {
+		return "", fmt.Errorf("supplied k8s version is not supported: %v", k8sVersion)
+	}
+
+	return k8sVersion, nil
+}
+
 // joinTokenGetter returns Kubernetes bootstrap (join) tokens.
 type joinTokenGetter interface {
 	// GetJoinToken returns a bootstrap (join) token.

joinservice/internal/server/server_test.go

@@ -37,6 +37,7 @@ func TestIssueJoinTicket(t *testing.T) {
 		CACertHashes:      []string{"hash"},
 		Token:             "token",
 	}
+	testK8sVersion := "1.23.6"
 
 	testCases := map[string]struct {
 		isControlPlane bool
@@ -115,6 +116,10 @@ func TestIssueJoinTicket(t *testing.T) {
 			if len(tc.id) > 0 {
 				require.NoError(file.Write(filepath.Join(constants.ServiceBasePath, constants.IDFilename), tc.id, 0o644))
 			}
+
+			// IssueJoinTicket tries to read the k8s-version ConfigMap from a mounted file.
+			require.NoError(file.Write(filepath.Join(constants.ServiceBasePath, constants.K8sVersion), []byte(testK8sVersion), 0o644))
+
 			api := New(
 				logger.NewTest(t),
 				file,

joinservice/joinproto/join.pb.go

@@ -96,6 +96,7 @@ type IssueJoinTicketResponse struct {
 	Token                    string                   `protobuf:"bytes,6,opt,name=token,proto3" json:"token,omitempty"`
 	DiscoveryTokenCaCertHash string                   `protobuf:"bytes,7,opt,name=discovery_token_ca_cert_hash,json=discoveryTokenCaCertHash,proto3" json:"discovery_token_ca_cert_hash,omitempty"`
 	ControlPlaneFiles        []*ControlPlaneCertOrKey `protobuf:"bytes,8,rep,name=control_plane_files,json=controlPlaneFiles,proto3" json:"control_plane_files,omitempty"`
+	KubernetesVersion        string                   `protobuf:"bytes,9,opt,name=kubernetes_version,json=kubernetesVersion,proto3" json:"kubernetes_version,omitempty"`
 }
 
 func (x *IssueJoinTicketResponse) Reset() {
@@ -186,6 +187,13 @@ func (x *IssueJoinTicketResponse) GetControlPlaneFiles() []*ControlPlaneCertOrKe
 	return nil
 }
 
+func (x *IssueJoinTicketResponse) GetKubernetesVersion() string {
+	if x != nil {
+		return x.KubernetesVersion
+	}
+	return ""
+}
+
 type ControlPlaneCertOrKey struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -254,7 +262,7 @@ var file_join_proto_rawDesc = []byte{
 	0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x69,
 	0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x18,
 	0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x69, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c,
-	0x50, 0x6c, 0x61, 0x6e, 0x65, 0x22, 0xf3, 0x02, 0x0a, 0x17, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a,
+	0x50, 0x6c, 0x61, 0x6e, 0x65, 0x22, 0xa2, 0x03, 0x0a, 0x17, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a,
 	0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
 	0x65, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x64, 0x69, 0x73, 0x6b, 0x5f,
 	0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x73, 0x74, 0x61, 0x74, 0x65,
@@ -277,21 +285,24 @@ var file_join_proto_rawDesc = []byte{
 	0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x63,
 	0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x5f, 0x63, 0x65, 0x72,
 	0x74, 0x5f, 0x6f, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f,
-	0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x22, 0x43, 0x0a, 0x19, 0x63,
+	0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x6b,
-	0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x5f, 0x63, 0x65, 0x72,
+	0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
-	0x74, 0x5f, 0x6f, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65,
+	0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04,
+	0x74, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x43, 0x0a, 0x19, 0x63, 0x6f,
-	0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61,
+	0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x5f, 0x63, 0x65, 0x72, 0x74,
-	0x32, 0x55, 0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, 0x4e, 0x0a, 0x0f, 0x49, 0x73, 0x73, 0x75, 0x65,
+	0x5f, 0x6f, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
-	0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1c, 0x2e, 0x6a, 0x6f, 0x69,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64,
-	0x6e, 0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65,
+	0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x32,
-	0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e,
+	0x55, 0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, 0x4e, 0x0a, 0x0f, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a,
-	0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52,
+	0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1c, 0x2e, 0x6a, 0x6f, 0x69, 0x6e,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75,
+	0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74,
-	0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x49,
-	0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f,
+	0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65,
-	0x6a, 0x6f, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x6a, 0x6f, 0x69, 0x6e,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73,
+	0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a,
+	0x6f, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x6a, 0x6f, 0x69, 0x6e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

joinservice/joinproto/join.proto

@@ -24,6 +24,7 @@ message IssueJoinTicketResponse {
     string token = 6;
     string discovery_token_ca_cert_hash = 7;
     repeated control_plane_cert_or_key control_plane_files = 8;
+    string kubernetes_version = 9;
 }
 
 message control_plane_cert_or_key {