From a68ee817fff796c9e7a1460c1bab5c180a349f82 Mon Sep 17 00:00:00 2001 From: Otto Bittner Date: Mon, 18 Jul 2022 12:28:02 +0200 Subject: [PATCH] AB#2074: Choosable K8S Version (#277) AB#2074: Add configurable k8s version Configurable version flow: * cli config holds/validates k8sVersion * InitCluster receive a k8sVersion arg * InitCluster creates CM "k8s-version" * kubeadm's InitConfiguration receives k8sVersion * joinservice spec mounts/reads k8s-version CM * joinservice supplies k8sVersion via JoinTicketResponse Other changes: * Remove unused test code (FakeK8SClient) * move VersionConfig map to /internal/versions * installk8sComponents is now a function instead of a method --- bootstrapper/cmd/bootstrapper/test.go | 2 +- bootstrapper/internal/joinclient/client.go | 3 +- .../internal/joinclient/client_test.go | 2 +- .../internal/kubernetes/k8sapi/constants.go | 16 ++++ .../kubernetes/k8sapi/kubeadm_config.go | 7 +- .../kubernetes/k8sapi/kubeadm_config_test.go | 4 +- .../k8sapi/kubectl/client/client.go | 12 +++ .../kubernetes/k8sapi/kubectl/kubectl.go | 17 ++++ .../kubernetes/k8sapi/kubectl/kubectl_test.go | 23 +++-- .../k8sapi/resources/access_manager.go | 3 +- .../k8sapi/resources/cluster_autoscaler.go | 3 +- .../k8sapi/resources/gcp_guest_agent.go | 3 +- .../kubernetes/k8sapi/resources/images.go | 13 --- .../k8sapi/resources/joinservice.go | 22 ++++- .../kubernetes/k8sapi/resources/kms.go | 3 +- .../k8sapi/resources/verification.go | 3 +- .../internal/kubernetes/k8sapi/util.go | 54 ++++++++++- .../internal/kubernetes/k8sapi/versions.go | 95 ------------------- .../internal/kubernetes/kubernetes.go | 69 +++++++++----- .../internal/kubernetes/kubernetes_test.go | 56 ++++++++--- cli/internal/cmd/init.go | 2 +- hack/go.mod | 1 + hack/go.sum | 4 + internal/config/config.go | 14 +++ internal/constants/constants.go | 4 +- internal/versions/containerImages.go | 13 +++ internal/versions/k8sBinaries.go | 31 ++++++ joinservice/internal/server/server.go | 23 +++++ joinservice/internal/server/server_test.go | 5 + joinservice/joinproto/join.pb.go | 43 +++++---- joinservice/joinproto/join.proto | 1 + 31 files changed, 360 insertions(+), 191 deletions(-) create mode 100644 bootstrapper/internal/kubernetes/k8sapi/constants.go delete mode 100644 bootstrapper/internal/kubernetes/k8sapi/resources/images.go delete mode 100644 bootstrapper/internal/kubernetes/k8sapi/versions.go create mode 100644 internal/versions/containerImages.go create mode 100644 internal/versions/k8sBinaries.go diff --git a/bootstrapper/cmd/bootstrapper/test.go b/bootstrapper/cmd/bootstrapper/test.go index 0e3b95b1f..04225601f 100644 --- a/bootstrapper/cmd/bootstrapper/test.go +++ b/bootstrapper/cmd/bootstrapper/test.go @@ -21,7 +21,7 @@ func (c *clusterFake) InitCluster(context.Context, []string, string, string, att } // JoinCluster will fake joining the current node to an existing cluster. -func (c *clusterFake) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, *logger.Logger) error { +func (c *clusterFake) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, string, *logger.Logger) error { return nil } diff --git a/bootstrapper/internal/joinclient/client.go b/bootstrapper/internal/joinclient/client.go index 68d9824c8..144eac252 100644 --- a/bootstrapper/internal/joinclient/client.go +++ b/bootstrapper/internal/joinclient/client.go @@ -272,7 +272,7 @@ func (c *JoinClient) startNodeAndJoin(ticket *joinproto.IssueJoinTicketResponse, Token: ticket.Token, CACertHashes: []string{ticket.DiscoveryTokenCaCertHash}, } - if err := c.joiner.JoinCluster(ctx, btd, c.role, c.log); err != nil { + if err := c.joiner.JoinCluster(ctx, btd, c.role, ticket.KubernetesVersion, c.log); err != nil { return fmt.Errorf("joining Kubernetes cluster: %w", err) } @@ -386,6 +386,7 @@ type ClusterJoiner interface { ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, + k8sVersion string, log *logger.Logger, ) error } diff --git a/bootstrapper/internal/joinclient/client_test.go b/bootstrapper/internal/joinclient/client_test.go index d2600547d..01df71224 100644 --- a/bootstrapper/internal/joinclient/client_test.go +++ b/bootstrapper/internal/joinclient/client_test.go @@ -385,7 +385,7 @@ type stubClusterJoiner struct { joinClusterErr error } -func (j *stubClusterJoiner) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, *logger.Logger) error { +func (j *stubClusterJoiner) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, string, *logger.Logger) error { j.joinClusterCalled = true return j.joinClusterErr } diff --git a/bootstrapper/internal/kubernetes/k8sapi/constants.go b/bootstrapper/internal/kubernetes/k8sapi/constants.go new file mode 100644 index 000000000..4c355ee38 --- /dev/null +++ b/bootstrapper/internal/kubernetes/k8sapi/constants.go @@ -0,0 +1,16 @@ +package k8sapi + +const ( + // Paths and permissions necessary for Kubernetes installation. + cniPluginsDir = "/opt/cni/bin" + binDir = "/run/state/bin" + kubeadmPath = "/run/state/bin/kubeadm" + kubeletPath = "/run/state/bin/kubelet" + kubectlPath = "/run/state/bin/kubectl" + kubeletServiceEtcPath = "/etc/systemd/system/kubelet.service" + kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service" + kubeadmConfEtcPath = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" + kubeadmConfStatePath = "/run/state/systemd/system/kubelet.service.d/10-kubeadm.conf" + executablePerm = 0o544 + systemdUnitPerm = 0o644 +) diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go index a4bb41f19..1a23f1a5d 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go @@ -5,7 +5,6 @@ import ( "github.com/edgelesssys/constellation/bootstrapper/internal/kubelet" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" - "github.com/edgelesssys/constellation/internal/constants" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconf "k8s.io/kubelet/config/v1beta1" @@ -25,7 +24,7 @@ const ( type CoreOSConfiguration struct{} -func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) KubeadmInitYAML { +func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sVersion string) KubeadmInitYAML { var cloudProvider string if externalCloudProvider { cloudProvider = "external" @@ -48,12 +47,14 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube BindPort: bindPort, }, }, + // https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3#ClusterConfiguration ClusterConfiguration: kubeadm.ClusterConfiguration{ TypeMeta: metav1.TypeMeta{ Kind: "ClusterConfiguration", APIVersion: kubeadm.SchemeGroupVersion.String(), }, - KubernetesVersion: constants.KubernetesVersion, + // Target kubernetes version of the control plane. + KubernetesVersion: k8sVersion, // necessary to be able to access the kubeapi server through localhost APIServer: kubeadm.APIServer{ ControlPlaneComponent: kubeadm.ControlPlaneComponent{ diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go index bdf5caa4e..28538fabe 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go @@ -19,11 +19,11 @@ func TestInitConfiguration(t *testing.T) { config KubeadmInitYAML }{ "CoreOS init config can be created": { - config: coreOSConfig.InitConfiguration(true), + config: coreOSConfig.InitConfiguration(true, "3.2.1"), }, "CoreOS init config with all fields can be created": { config: func() KubeadmInitYAML { - c := coreOSConfig.InitConfiguration(true) + c := coreOSConfig.InitConfiguration(true, "3.2.1") c.SetAPIServerAdvertiseAddress("192.0.2.0") c.SetNodeIP("192.0.2.0") c.SetNodeName("node") diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go b/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go index 4b3e80e93..3227d6919 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go @@ -2,9 +2,11 @@ package client import ( "bytes" + "context" "fmt" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -88,3 +90,13 @@ func (c *Client) GetObjects(resources resources.Marshaler) ([]*resource.Info, er Do() return result.Infos() } + +// CreateConfigMap creates the given ConfigMap. +func (c *Client) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error { + _, err := c.clientset.CoreV1().ConfigMaps(configMap.ObjectMeta.Namespace).Create(ctx, &configMap, metav1.CreateOptions{}) + if err != nil { + return err + } + + return nil +} diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go b/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go index 3a5b97c9c..cc7eed6a2 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go @@ -1,10 +1,12 @@ package kubectl import ( + "context" "errors" "fmt" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" + corev1 "k8s.io/api/core/v1" "k8s.io/cli-runtime/pkg/resource" ) @@ -17,6 +19,7 @@ type Client interface { ApplyOneObject(info *resource.Info, forceConflicts bool) error // GetObjects converts resources into prepared info fields for use in ApplyOneObject. GetObjects(resources resources.Marshaler) ([]*resource.Info, error) + CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error } // clientGenerator can generate new clients from a kubeconfig. @@ -66,3 +69,17 @@ func (k *Kubectl) Apply(resources resources.Marshaler, forceConflicts bool) erro func (k *Kubectl) SetKubeconfig(kubeconfig []byte) { k.kubeconfig = kubeconfig } + +func (k *Kubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error { + client, err := k.clientGenerator.NewClient(k.kubeconfig) + if err != nil { + return err + } + + err = client.CreateConfigMap(ctx, configMap) + if err != nil { + return err + } + + return nil +} diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go b/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go index 52d9530c7..738d34dc3 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go @@ -1,12 +1,14 @@ package kubectl import ( + "context" "errors" "testing" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/stretchr/testify/assert" "go.uber.org/goleak" + corev1 "k8s.io/api/core/v1" "k8s.io/cli-runtime/pkg/resource" ) @@ -15,9 +17,10 @@ func TestMain(m *testing.M) { } type stubClient struct { - applyOneObjectErr error - getObjectsInfos []*resource.Info - getObjectsErr error + applyOneObjectErr error + getObjectsInfos []*resource.Info + getObjectsErr error + createConfigMapErr error } func (s *stubClient) ApplyOneObject(info *resource.Info, forceConflicts bool) error { @@ -28,11 +31,16 @@ func (s *stubClient) GetObjects(resources resources.Marshaler) ([]*resource.Info return s.getObjectsInfos, s.getObjectsErr } +func (s *stubClient) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error { + return s.createConfigMapErr +} + type stubClientGenerator struct { - applyOneObjectErr error - getObjectsInfos []*resource.Info - getObjectsErr error - newClientErr error + applyOneObjectErr error + getObjectsInfos []*resource.Info + getObjectsErr error + newClientErr error + createConfigMapErr error } func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) { @@ -40,6 +48,7 @@ func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) { s.applyOneObjectErr, s.getObjectsInfos, s.getObjectsErr, + s.createConfigMapErr, }, s.newClientErr } diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/access_manager.go b/bootstrapper/internal/kubernetes/k8sapi/resources/access_manager.go index 283301500..14c0adf45 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/access_manager.go +++ b/bootstrapper/internal/kubernetes/k8sapi/resources/access_manager.go @@ -2,6 +2,7 @@ package resources import ( "github.com/edgelesssys/constellation/internal/secrets" + "github.com/edgelesssys/constellation/internal/versions" "google.golang.org/protobuf/proto" apps "k8s.io/api/apps/v1" k8s "k8s.io/api/core/v1" @@ -104,7 +105,7 @@ func NewAccessManagerDeployment(sshUsers map[string]string) *accessManagerDeploy InitContainers: []k8s.Container{ { Name: "constellation-access-manager", - Image: accessManagerImage, + Image: versions.AccessManagerImage, VolumeMounts: []k8s.VolumeMount{ { Name: "host", diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go b/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go index 6ffca5c0f..5c3203452 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go +++ b/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go @@ -1,6 +1,7 @@ package resources import ( + "github.com/edgelesssys/constellation/internal/versions" "google.golang.org/protobuf/proto" apps "k8s.io/api/apps/v1" k8s "k8s.io/api/core/v1" @@ -434,7 +435,7 @@ func NewDefaultAutoscalerDeployment(extraVolumes []k8s.Volume, extraVolumeMounts Containers: []k8s.Container{ { Name: "cluster-autoscaler", - Image: clusterAutoscalerImage, + Image: versions.ClusterAutoscalerImage, ImagePullPolicy: k8s.PullIfNotPresent, LivenessProbe: &k8s.Probe{ ProbeHandler: k8s.ProbeHandler{ diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/gcp_guest_agent.go b/bootstrapper/internal/kubernetes/k8sapi/resources/gcp_guest_agent.go index 019256443..23ed1f8e2 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/gcp_guest_agent.go +++ b/bootstrapper/internal/kubernetes/k8sapi/resources/gcp_guest_agent.go @@ -2,6 +2,7 @@ package resources import ( "github.com/edgelesssys/constellation/internal/secrets" + "github.com/edgelesssys/constellation/internal/versions" apps "k8s.io/api/apps/v1" k8s "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -64,7 +65,7 @@ func NewGCPGuestAgentDaemonset() *gcpGuestAgentDaemonset { Containers: []k8s.Container{ { Name: "gcp-guest-agent", - Image: gcpGuestImage, // built from https://github.com/edgelesssys/gcp-guest-agent + Image: versions.GcpGuestImage, // built from https://github.com/edgelesssys/gcp-guest-agent SecurityContext: &k8s.SecurityContext{ Privileged: func(b bool) *bool { return &b }(true), Capabilities: &k8s.Capabilities{ diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/images.go b/bootstrapper/internal/kubernetes/k8sapi/resources/images.go deleted file mode 100644 index 0e1d1947a..000000000 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/images.go +++ /dev/null @@ -1,13 +0,0 @@ -package resources - -const ( - // Constellation images. - joinImage = "ghcr.io/edgelesssys/constellation/join-service:v1.3.2-0.20220714151638-d295be31" - accessManagerImage = "ghcr.io/edgelesssys/constellation/access-manager:v1.3.2-0.20220714151638-d295be31" - kmsImage = "ghcr.io/edgelesssys/constellation/kmsserver:v1.3.2-0.20220714151638-d295be31" - verificationImage = "ghcr.io/edgelesssys/constellation/verification-service:v1.3.2-0.20220714151638-d295be31" - gcpGuestImage = "ghcr.io/edgelesssys/gcp-guest-agent:latest" - - // external images. - clusterAutoscalerImage = "k8s.gcr.io/autoscaling/cluster-autoscaler:v1.23.0" -) diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/joinservice.go b/bootstrapper/internal/kubernetes/k8sapi/resources/joinservice.go index 08db290d2..67d78602c 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/joinservice.go +++ b/bootstrapper/internal/kubernetes/k8sapi/resources/joinservice.go @@ -5,6 +5,7 @@ import ( "github.com/edgelesssys/constellation/internal/constants" "github.com/edgelesssys/constellation/internal/secrets" + "github.com/edgelesssys/constellation/internal/versions" apps "k8s.io/api/apps/v1" k8s "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" @@ -135,7 +136,7 @@ func NewJoinServiceDaemonset(csp string, measurementsJSON, idJSON string) *joinS Containers: []k8s.Container{ { Name: "join-service", - Image: joinImage, + Image: versions.JoinImage, Ports: []k8s.ContainerPort{ { ContainerPort: constants.JoinServicePort, @@ -167,9 +168,22 @@ func NewJoinServiceDaemonset(csp string, measurementsJSON, idJSON string) *joinS { Name: "config", VolumeSource: k8s.VolumeSource{ - ConfigMap: &k8s.ConfigMapVolumeSource{ - LocalObjectReference: k8s.LocalObjectReference{ - Name: "join-config", + Projected: &k8s.ProjectedVolumeSource{ + Sources: []k8s.VolumeProjection{ + { + ConfigMap: &k8s.ConfigMapProjection{ + LocalObjectReference: k8s.LocalObjectReference{ + Name: "join-config", + }, + }, + }, + { + ConfigMap: &k8s.ConfigMapProjection{ + LocalObjectReference: k8s.LocalObjectReference{ + Name: constants.K8sVersion, + }, + }, + }, }, }, }, diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/kms.go b/bootstrapper/internal/kubernetes/k8sapi/resources/kms.go index 36ae4be5d..a9f20e477 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/kms.go +++ b/bootstrapper/internal/kubernetes/k8sapi/resources/kms.go @@ -5,6 +5,7 @@ import ( "github.com/edgelesssys/constellation/internal/constants" "github.com/edgelesssys/constellation/internal/secrets" + "github.com/edgelesssys/constellation/internal/versions" apps "k8s.io/api/apps/v1" k8s "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" @@ -226,7 +227,7 @@ func NewKMSDeployment(csp string, masterSecret []byte) *kmsDeployment { Containers: []k8s.Container{ { Name: "kms", - Image: kmsImage, + Image: versions.KmsImage, Args: []string{ fmt.Sprintf("--atls-port=%d", constants.KMSATLSPort), fmt.Sprintf("--port=%d", constants.KMSPort), diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/verification.go b/bootstrapper/internal/kubernetes/k8sapi/resources/verification.go index d9ac0928e..f691a6a9b 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/verification.go +++ b/bootstrapper/internal/kubernetes/k8sapi/resources/verification.go @@ -5,6 +5,7 @@ import ( "github.com/edgelesssys/constellation/internal/constants" "github.com/edgelesssys/constellation/internal/secrets" + "github.com/edgelesssys/constellation/internal/versions" apps "k8s.io/api/apps/v1" k8s "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,7 +74,7 @@ func NewVerificationDaemonSet(csp string) *verificationDaemonset { Containers: []k8s.Container{ { Name: "verification-service", - Image: verificationImage, + Image: versions.VerificationImage, Ports: []k8s.ContainerPort{ { Name: "http", diff --git a/bootstrapper/internal/kubernetes/k8sapi/util.go b/bootstrapper/internal/kubernetes/k8sapi/util.go index 0dde9c2ef..0f4f5f0cd 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/util.go +++ b/bootstrapper/internal/kubernetes/k8sapi/util.go @@ -7,6 +7,7 @@ import ( "encoding/pem" "errors" "fmt" + "io/fs" "net" "net/http" "os" @@ -21,8 +22,12 @@ import ( "github.com/edgelesssys/constellation/bootstrapper/util" "github.com/edgelesssys/constellation/internal/file" "github.com/edgelesssys/constellation/internal/logger" + "github.com/edgelesssys/constellation/internal/versions" + "github.com/icholy/replace" "github.com/spf13/afero" "go.uber.org/zap" + "golang.org/x/text/transform" + corev1 "k8s.io/api/core/v1" "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) @@ -39,9 +44,17 @@ var providerIDRegex = regexp.MustCompile(`^azure:///subscriptions/([^/]+)/resour type Client interface { Apply(resources resources.Marshaler, forceConflicts bool) error SetKubeconfig(kubeconfig []byte) + CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error // TODO: add tolerations } +type installer interface { + Install( + ctx context.Context, sourceURL string, destinations []string, perm fs.FileMode, + extract bool, transforms ...transform.Transformer, + ) error +} + // KubernetesUtil provides low level management of the kubernetes cluster. type KubernetesUtil struct { inst installer @@ -58,13 +71,46 @@ func NewKubernetesUtil() *KubernetesUtil { // InstallComponents installs kubernetes components in the version specified. func (k *KubernetesUtil) InstallComponents(ctx context.Context, version string) error { - var versionConf kubernetesVersion + var versionConf versions.KubernetesVersion var ok bool - if versionConf, ok = versionConfigs[version]; !ok { + if versionConf, ok = versions.VersionConfigs[version]; !ok { return fmt.Errorf("unsupported kubernetes version %q", version) } - if err := versionConf.installK8sComponents(ctx, k.inst); err != nil { - return err + + if err := k.inst.Install( + ctx, versionConf.CNIPluginsURL, []string{cniPluginsDir}, executablePerm, true, + ); err != nil { + return fmt.Errorf("installing cni plugins: %w", err) + } + if err := k.inst.Install( + ctx, versionConf.CrictlURL, []string{binDir}, executablePerm, true, + ); err != nil { + return fmt.Errorf("installing crictl: %w", err) + } + if err := k.inst.Install( + ctx, versionConf.KubeletServiceURL, []string{kubeletServiceEtcPath, kubeletServiceStatePath}, systemdUnitPerm, false, replace.String("/usr/bin", binDir), + ); err != nil { + return fmt.Errorf("installing kubelet service: %w", err) + } + if err := k.inst.Install( + ctx, versionConf.KubeadmConfURL, []string{kubeadmConfEtcPath, kubeadmConfStatePath}, systemdUnitPerm, false, replace.String("/usr/bin", binDir), + ); err != nil { + return fmt.Errorf("installing kubeadm conf: %w", err) + } + if err := k.inst.Install( + ctx, versionConf.KubeletURL, []string{kubeletPath}, executablePerm, false, + ); err != nil { + return fmt.Errorf("installing kubelet: %w", err) + } + if err := k.inst.Install( + ctx, versionConf.KubeadmURL, []string{kubeadmPath}, executablePerm, false, + ); err != nil { + return fmt.Errorf("installing kubeadm: %w", err) + } + if err := k.inst.Install( + ctx, versionConf.KubectlURL, []string{kubectlPath}, executablePerm, false, + ); err != nil { + return fmt.Errorf("installing kubectl: %w", err) } return enableSystemdUnit(ctx, kubeletServiceEtcPath) diff --git a/bootstrapper/internal/kubernetes/k8sapi/versions.go b/bootstrapper/internal/kubernetes/k8sapi/versions.go deleted file mode 100644 index dca4a4d16..000000000 --- a/bootstrapper/internal/kubernetes/k8sapi/versions.go +++ /dev/null @@ -1,95 +0,0 @@ -package k8sapi - -import ( - "context" - "fmt" - "io/fs" - - "github.com/icholy/replace" - "golang.org/x/text/transform" -) - -const ( - cniPluginsDir = "/opt/cni/bin" - binDir = "/run/state/bin" - kubeadmPath = "/run/state/bin/kubeadm" - kubeletPath = "/run/state/bin/kubelet" - kubectlPath = "/run/state/bin/kubectl" - kubeletServiceEtcPath = "/etc/systemd/system/kubelet.service" - kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service" - kubeadmConfEtcPath = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" - kubeadmConfStatePath = "/run/state/systemd/system/kubelet.service.d/10-kubeadm.conf" - executablePerm = 0o544 - systemdUnitPerm = 0o644 -) - -// versionConfigs holds download URLs for all required kubernetes components for every supported version. -var versionConfigs map[string]kubernetesVersion = map[string]kubernetesVersion{ - "1.23.6": { - CNIPluginsURL: "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz", - CrictlURL: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.1/crictl-v1.24.1-linux-amd64.tar.gz", - KubeletServiceURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service", - KubeadmConfURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf", - KubeletURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubelet", - KubeadmURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubeadm", - KubectlURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl", - }, -} - -type kubernetesVersion struct { - CNIPluginsURL string - CrictlURL string - KubeletServiceURL string - KubeadmConfURL string - KubeletURL string - KubeadmURL string - KubectlURL string -} - -// installK8sComponents installs kubernetes components for this version. -// reference: https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#installing-kubeadm-kubelet-and-kubectl . -func (k *kubernetesVersion) installK8sComponents(ctx context.Context, inst installer) error { - if err := inst.Install( - ctx, k.CNIPluginsURL, []string{cniPluginsDir}, executablePerm, true, - ); err != nil { - return fmt.Errorf("installing cni plugins: %w", err) - } - if err := inst.Install( - ctx, k.CrictlURL, []string{binDir}, executablePerm, true, - ); err != nil { - return fmt.Errorf("installing crictl: %w", err) - } - if err := inst.Install( - ctx, k.KubeletServiceURL, []string{kubeletServiceEtcPath, kubeletServiceStatePath}, systemdUnitPerm, false, replace.String("/usr/bin", binDir), - ); err != nil { - return fmt.Errorf("installing kubelet service: %w", err) - } - if err := inst.Install( - ctx, k.KubeadmConfURL, []string{kubeadmConfEtcPath, kubeadmConfStatePath}, systemdUnitPerm, false, replace.String("/usr/bin", binDir), - ); err != nil { - return fmt.Errorf("installing kubeadm conf: %w", err) - } - if err := inst.Install( - ctx, k.KubeletURL, []string{kubeletPath}, executablePerm, false, - ); err != nil { - return fmt.Errorf("installing kubelet: %w", err) - } - if err := inst.Install( - ctx, k.KubeadmURL, []string{kubeadmPath}, executablePerm, false, - ); err != nil { - return fmt.Errorf("installing kubeadm: %w", err) - } - if err := inst.Install( - ctx, k.KubectlURL, []string{kubectlPath}, executablePerm, false, - ); err != nil { - return fmt.Errorf("installing kubectl: %w", err) - } - return nil -} - -type installer interface { - Install( - ctx context.Context, sourceURL string, destinations []string, perm fs.FileMode, - extract bool, transforms ...transform.Transformer, - ) error -} diff --git a/bootstrapper/internal/kubernetes/kubernetes.go b/bootstrapper/internal/kubernetes/kubernetes.go index d13d43d40..56ccc7d80 100644 --- a/bootstrapper/internal/kubernetes/kubernetes.go +++ b/bootstrapper/internal/kubernetes/kubernetes.go @@ -16,8 +16,11 @@ import ( attestationtypes "github.com/edgelesssys/constellation/internal/attestation/types" "github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/edgelesssys/constellation/internal/logger" + "github.com/edgelesssys/constellation/internal/versions" "github.com/spf13/afero" "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" ) @@ -28,7 +31,7 @@ type configReader interface { // configurationProvider provides kubeadm init and join configuration. type configurationProvider interface { - InitConfiguration(externalCloudProvider bool) k8sapi.KubeadmInitYAML + InitConfiguration(externalCloudProvider bool, k8sVersion string) k8sapi.KubeadmInitYAML JoinConfiguration(externalCloudProvider bool) k8sapi.KubeadmJoinYAML } @@ -79,7 +82,6 @@ func (k *KubeWrapper) InitCluster( ctx context.Context, autoscalingNodeGroups []string, cloudServiceAccountURI, k8sVersion string, id attestationtypes.ID, kmsConfig KMSConfig, sshUsers map[string]string, log *logger.Logger, ) ([]byte, error) { - // TODO: k8s version should be user input log.With(zap.String("version", k8sVersion)).Infof("Installing Kubernetes components") if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil { return nil, err @@ -149,7 +151,7 @@ func (k *KubeWrapper) InitCluster( ).Infof("Setting information for node") // Step 2: configure kubeadm init config - initConfig := k.configProvider.InitConfiguration(k.cloudControllerManager.Supported()) + initConfig := k.configProvider.InitConfiguration(k.cloudControllerManager.Supported(), k8sVersion) initConfig.SetNodeIP(nodeIP) initConfig.SetCertSANs([]string{publicIP, nodeIP}) initConfig.SetNodeName(nodeName) @@ -219,16 +221,21 @@ func (k *KubeWrapper) InitCluster( } } + // Store the received k8sVersion in a ConfigMap, overwriting exisiting values (there shouldn't be any). + // Joining nodes determine the kubernetes version they will install based on this ConfigMap. + if err := k.setupK8sVersionConfigMap(ctx, k8sVersion, true); err != nil { + return nil, fmt.Errorf("failed to setup k8s version ConfigMap: %v", err) + } + k.clusterUtil.FixCilium(nodeName) return k.GetKubeconfig() } // JoinCluster joins existing Kubernetes cluster. -func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, log *logger.Logger) error { - // TODO: k8s version should be user input - log.With(zap.String("version", "1.23.6")).Infof("Installing Kubernetes components") - if err := k.clusterUtil.InstallComponents(ctx, "1.23.6"); err != nil { +func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, k8sVersion string, log *logger.Logger) error { + log.With(zap.String("version", k8sVersion)).Infof("Installing Kubernetes components") + if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil { return err } @@ -357,6 +364,35 @@ func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata, return nil } +// setupK8sVersionConfigMap applies a ConfigMap (cf. server-side apply) to consistently store the installed k8s version. +func (k *KubeWrapper) setupK8sVersionConfigMap(ctx context.Context, k8sVersion string, forceConflicts bool) error { + if !versions.IsSupportedK8sVersion(k8sVersion) { + return fmt.Errorf("supplied k8s version is not supported: %v", k8sVersion) + } + + config := corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "k8s-version", + Namespace: "kube-system", + }, + Data: map[string]string{ + "K8sVersion": k8sVersion, + }, + } + + // We do not use the client's Apply method here since we are handling a kubernetes-native type. + // These types don't implement our custom Marshaler interface. + if err := k.client.CreateConfigMap(ctx, config); err != nil { + return fmt.Errorf("Apply in KubeWrapper.setupK8sVersionConfigMap(..) failed with: %v", err) + } + + return nil +} + // manuallySetLoadbalancerIP sets the loadbalancer IP of the first control plane during init. // The GCP guest agent does this usually, but is deployed in the cluster that doesn't exist // at this point. This is a workaround to set the loadbalancer IP manually, so kubeadm and kubelet @@ -391,22 +427,3 @@ func k8sCompliantHostname(in string) string { func (k *KubeWrapper) StartKubelet() error { return k.clusterUtil.StartKubelet() } - -type fakeK8SClient struct { - kubeconfig []byte -} - -// NewFakeK8SClient creates a new, fake k8s client where apply always works. -func NewFakeK8SClient([]byte) (k8sapi.Client, error) { - return &fakeK8SClient{}, nil -} - -// Apply fakes applying Kubernetes resources. -func (f *fakeK8SClient) Apply(resources resources.Marshaler, forceConflicts bool) error { - return nil -} - -// SetKubeconfig stores the kubeconfig given to it. -func (f *fakeK8SClient) SetKubeconfig(kubeconfig []byte) { - f.kubeconfig = kubeconfig -} diff --git a/bootstrapper/internal/kubernetes/kubernetes_test.go b/bootstrapper/internal/kubernetes/kubernetes_test.go index 0aee80bcb..c53646d43 100644 --- a/bootstrapper/internal/kubernetes/kubernetes_test.go +++ b/bootstrapper/internal/kubernetes/kubernetes_test.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" + corev1 "k8s.io/api/core/v1" kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" ) @@ -35,11 +36,10 @@ func TestInitCluster(t *testing.T) { publicIP := "192.0.2.2" loadbalancerIP := "192.0.2.3" aliasIPRange := "192.0.2.0/24" - k8sVersion := "1.23.8" testCases := map[string]struct { clusterUtil stubClusterUtil - kubeCTL stubKubeCTL + kubectl stubKubectl providerMetadata ProviderMetadata CloudControllerManager CloudControllerManager CloudNodeManager CloudNodeManager @@ -47,6 +47,7 @@ func TestInitCluster(t *testing.T) { kubeconfigReader configReader wantConfig k8sapi.KubeadmInitYAML wantErr bool + k8sVersion string }{ "kubeadm init works without metadata": { clusterUtil: stubClusterUtil{}, @@ -69,6 +70,7 @@ func TestInitCluster(t *testing.T) { }, ClusterConfiguration: kubeadm.ClusterConfiguration{}, }, + k8sVersion: "1.23.6", }, "kubeadm init works with metadata and loadbalancer": { clusterUtil: stubClusterUtil{}, @@ -107,7 +109,8 @@ func TestInitCluster(t *testing.T) { }, }, }, - wantErr: false, + wantErr: false, + k8sVersion: "1.23.6", }, "kubeadm init fails when retrieving metadata self": { clusterUtil: stubClusterUtil{}, @@ -122,6 +125,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, + k8sVersion: "1.23.6", }, "kubeadm init fails when retrieving metadata subnetwork cidr": { clusterUtil: stubClusterUtil{}, @@ -136,6 +140,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, + k8sVersion: "1.23.6", }, "kubeadm init fails when retrieving metadata loadbalancer ip": { clusterUtil: stubClusterUtil{}, @@ -151,6 +156,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, + k8sVersion: "1.23.6", }, "kubeadm init fails when applying the init config": { clusterUtil: stubClusterUtil{initClusterErr: someErr}, @@ -162,6 +168,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, + k8sVersion: "1.23.6", }, "kubeadm init fails when setting up the pod network": { clusterUtil: stubClusterUtil{setupPodNetworkErr: someErr}, @@ -173,6 +180,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, + k8sVersion: "1.23.6", }, "kubeadm init fails when setting up the join service": { clusterUtil: stubClusterUtil{setupJoinServiceError: someErr}, @@ -184,6 +192,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, + k8sVersion: "1.23.6", }, "kubeadm init fails when setting the cloud contoller manager": { clusterUtil: stubClusterUtil{setupCloudControllerManagerError: someErr}, @@ -195,6 +204,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, + k8sVersion: "1.23.6", }, "kubeadm init fails when setting the cloud node manager": { clusterUtil: stubClusterUtil{setupCloudNodeManagerError: someErr}, @@ -206,6 +216,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{SupportedResp: true}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, + k8sVersion: "1.23.6", }, "kubeadm init fails when setting the cluster autoscaler": { clusterUtil: stubClusterUtil{setupAutoscalingError: someErr}, @@ -217,6 +228,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{SupportedResp: true}, wantErr: true, + k8sVersion: "1.23.6", }, "kubeadm init fails when reading kubeconfig": { clusterUtil: stubClusterUtil{}, @@ -228,6 +240,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, + k8sVersion: "1.23.6", }, "kubeadm init fails when setting up the kms": { clusterUtil: stubClusterUtil{setupKMSError: someErr}, @@ -239,6 +252,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{SupportedResp: false}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, + k8sVersion: "1.23.6", }, "kubeadm init fails when setting up verification service": { clusterUtil: stubClusterUtil{setupVerificationServiceErr: someErr}, @@ -250,6 +264,19 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{SupportedResp: false}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, + k8sVersion: "1.23.6", + }, + "unsupported k8sVersion fails cluster creation": { + clusterUtil: stubClusterUtil{}, + kubeconfigReader: &stubKubeconfigReader{ + Kubeconfig: []byte("someKubeconfig"), + }, + providerMetadata: &stubProviderMetadata{}, + CloudControllerManager: &stubCloudControllerManager{}, + CloudNodeManager: &stubCloudNodeManager{}, + ClusterAutoscaler: &stubClusterAutoscaler{}, + k8sVersion: "invalid version", + wantErr: true, }, } @@ -265,11 +292,11 @@ func TestInitCluster(t *testing.T) { cloudNodeManager: tc.CloudNodeManager, clusterAutoscaler: tc.ClusterAutoscaler, configProvider: &stubConfigProvider{InitConfig: k8sapi.KubeadmInitYAML{}}, - client: &tc.kubeCTL, + client: &tc.kubectl, kubeconfigReader: tc.kubeconfigReader, getIPAddr: func() (string, error) { return privateIP, nil }, } - _, err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountURI, k8sVersion, attestationtypes.ID{}, KMSConfig{MasterSecret: masterSecret}, nil, logger.NewTest(t)) + _, err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountURI, tc.k8sVersion, attestationtypes.ID{}, KMSConfig{MasterSecret: masterSecret}, nil, logger.NewTest(t)) if tc.wantErr { assert.Error(err) @@ -294,6 +321,8 @@ func TestJoinCluster(t *testing.T) { } privateIP := "192.0.2.1" + // stubClusterUtil does not validate the k8sVersion, thus it can be arbitrary. + k8sVersion := "3.2.1" testCases := map[string]struct { clusterUtil stubClusterUtil @@ -425,7 +454,7 @@ func TestJoinCluster(t *testing.T) { getIPAddr: func() (string, error) { return privateIP, nil }, } - err := kube.JoinCluster(context.Background(), joinCommand, tc.role, logger.NewTest(t)) + err := kube.JoinCluster(context.Background(), joinCommand, tc.role, k8sVersion, logger.NewTest(t)) if tc.wantErr { assert.Error(err) return @@ -559,7 +588,7 @@ type stubConfigProvider struct { JoinConfig k8sapi.KubeadmJoinYAML } -func (s *stubConfigProvider) InitConfiguration(_ bool) k8sapi.KubeadmInitYAML { +func (s *stubConfigProvider) InitConfiguration(_ bool, _ string) k8sapi.KubeadmInitYAML { return s.InitConfig } @@ -574,22 +603,27 @@ func (s *stubConfigProvider) JoinConfiguration(_ bool) k8sapi.KubeadmJoinYAML { return s.JoinConfig } -type stubKubeCTL struct { - ApplyErr error +type stubKubectl struct { + ApplyErr error + createConfigMapErr error resources []resources.Marshaler kubeconfigs [][]byte } -func (s *stubKubeCTL) Apply(resources resources.Marshaler, forceConflicts bool) error { +func (s *stubKubectl) Apply(resources resources.Marshaler, forceConflicts bool) error { s.resources = append(s.resources, resources) return s.ApplyErr } -func (s *stubKubeCTL) SetKubeconfig(kubeconfig []byte) { +func (s *stubKubectl) SetKubeconfig(kubeconfig []byte) { s.kubeconfigs = append(s.kubeconfigs, kubeconfig) } +func (s *stubKubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error { + return s.createConfigMapErr +} + type stubKubeconfigReader struct { Kubeconfig []byte ReadErr error diff --git a/cli/internal/cmd/init.go b/cli/internal/cmd/init.go index 53925e0cd..386dcd9e2 100644 --- a/cli/internal/cmd/init.go +++ b/cli/internal/cmd/init.go @@ -120,7 +120,7 @@ func initialize(cmd *cobra.Command, dialer grpcDialer, serviceAccCreator service KeyEncryptionKeyId: "", UseExistingKek: false, CloudServiceAccountUri: serviceAccount, - KubernetesVersion: "1.23.6", + KubernetesVersion: config.KubernetesVersion, SshUserKeys: ssh.ToProtoSlice(sshUsers), } resp, err := initCall(cmd.Context(), dialer, stat.BootstrapperHost, req) diff --git a/hack/go.mod b/hack/go.mod index 5a05371f0..17761c235 100644 --- a/hack/go.mod +++ b/hack/go.mod @@ -56,6 +56,7 @@ require ( github.com/emirpasic/gods v1.12.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/icholy/replace v0.5.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect diff --git a/hack/go.sum b/hack/go.sum index 0214beed9..2004b9e42 100644 --- a/hack/go.sum +++ b/hack/go.sum @@ -636,6 +636,8 @@ github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63 github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/icholy/replace v0.5.0 h1:Nx80zYQVlowdba+3Y6dvHDnmxaGtBrDlf2wYn9GyIXQ= +github.com/icholy/replace v0.5.0/go.mod h1:zzi8pxElj2t/5wHHHYmH45D+KxytX/t4w3ClY5nlK+g= github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -1344,6 +1346,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1654,6 +1657,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 h1:dbuHpmKjkDzSOMKAWl10QNlgaZUd3V1q99xc81tt2Kc= gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/config/config.go b/internal/config/config.go index 831c898c4..68e93d5b4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,7 @@ import ( "github.com/edgelesssys/constellation/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/internal/constants" "github.com/edgelesssys/constellation/internal/file" + "github.com/edgelesssys/constellation/internal/versions" "github.com/go-playground/locales/en" ut "github.com/go-playground/universal-translator" "github.com/go-playground/validator/v10" @@ -60,6 +61,9 @@ type Config struct { // examples: // - value: '[]UserKey{ { Username: "Alice", PublicKey: "ssh-rsa AAAAB3NzaC...5QXHKW1rufgtJeSeJ8= alice@domain.com" } }' SSHUsers []UserKey `yaml:"sshUsers,omitempty" validate:"dive"` + // description: | + // Kubernetes version installed in the cluster. + KubernetesVersion string `yaml:"kubernetesVersion" validate:"supported_k8s_version"` } // UserKey describes a user that should be created with corresponding public SSH key. @@ -226,9 +230,14 @@ func Default() *Config { Measurements: qemuPCRs, }, }, + KubernetesVersion: "1.23.6", } } +func validateK8sVersion(fl validator.FieldLevel) bool { + return versions.IsSupportedK8sVersion(fl.Field().String()) +} + // Validate checks the config values and returns validation error messages. // The function only returns an error if the validation itself fails. func (c *Config) Validate() ([]string, error) { @@ -238,6 +247,11 @@ func (c *Config) Validate() ([]string, error) { return nil, err } + // register custom validator with label supported_k8s_version to validate version based on available versionConfigs. + if err := validate.RegisterValidation("supported_k8s_version", validateK8sVersion); err != nil { + return nil, err + } + err := validate.Struct(c) if err == nil { return nil, nil diff --git a/internal/constants/constants.go b/internal/constants/constants.go index ec04d7763..4fde71131 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -63,12 +63,14 @@ const ( // Filenames for Constellation's micro services. // - // ServiceBasePath is the base path for the mounted micro services files. + // ServiceBasePath is the base path for the mounted micro service's files. ServiceBasePath = "/var/config" // MeasurementsFilename is the filename of CC measurements. MeasurementsFilename = "measurements" // IDFilename is the filename of Constellation's IDs. IDFilename = "id" + // K8sVersion is the filename of the mapped k8s-version configMap file. + K8sVersion = "k8s-version" // // Cryptographic constants. diff --git a/internal/versions/containerImages.go b/internal/versions/containerImages.go new file mode 100644 index 000000000..4556706da --- /dev/null +++ b/internal/versions/containerImages.go @@ -0,0 +1,13 @@ +package versions + +const ( + // Constellation images. + JoinImage = "ghcr.io/edgelesssys/constellation/join-service:v1.3.2-0.20220714151638-d295be31" + AccessManagerImage = "ghcr.io/edgelesssys/constellation/access-manager:v1.3.2-0.20220714151638-d295be31" + KmsImage = "ghcr.io/edgelesssys/constellation/kmsserver:v1.3.2-0.20220714151638-d295be31" + VerificationImage = "ghcr.io/edgelesssys/constellation/verification-service:v1.3.2-0.20220714151638-d295be31" + GcpGuestImage = "ghcr.io/edgelesssys/gcp-guest-agent:latest" + + // external images. + ClusterAutoscalerImage = "k8s.gcr.io/autoscaling/cluster-autoscaler:v1.23.0" +) diff --git a/internal/versions/k8sBinaries.go b/internal/versions/k8sBinaries.go new file mode 100644 index 000000000..b320b9f2e --- /dev/null +++ b/internal/versions/k8sBinaries.go @@ -0,0 +1,31 @@ +package versions + +// versionConfigs holds download URLs for all required kubernetes components for every supported version. +var VersionConfigs map[string]KubernetesVersion = map[string]KubernetesVersion{ + "1.23.6": { + CNIPluginsURL: "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz", + CrictlURL: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.1/crictl-v1.24.1-linux-amd64.tar.gz", + KubeletServiceURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service", + KubeadmConfURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf", + KubeletURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubelet", + KubeadmURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubeadm", + KubectlURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl", + }, +} + +type KubernetesVersion struct { + CNIPluginsURL string + CrictlURL string + KubeletServiceURL string + KubeadmConfURL string + KubeletURL string + KubeadmURL string + KubectlURL string +} + +func IsSupportedK8sVersion(version string) bool { + if _, ok := VersionConfigs[version]; !ok { + return false + } + return true +} diff --git a/joinservice/internal/server/server.go b/joinservice/internal/server/server.go index b882ceb47..bedf42142 100644 --- a/joinservice/internal/server/server.go +++ b/joinservice/internal/server/server.go @@ -12,6 +12,7 @@ import ( "github.com/edgelesssys/constellation/internal/file" "github.com/edgelesssys/constellation/internal/grpc/grpclog" "github.com/edgelesssys/constellation/internal/logger" + "github.com/edgelesssys/constellation/internal/versions" "github.com/edgelesssys/constellation/joinservice/joinproto" "go.uber.org/zap" "google.golang.org/grpc" @@ -92,6 +93,12 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi return nil, status.Errorf(codes.Internal, "unable to generate Kubernetes join arguments: %s", err) } + log.Infof("Querying K8sVersion ConfigMap") + k8sVersion, err := s.getK8sVersion(ctx) + if err != nil { + return nil, status.Errorf(codes.Internal, "unable to get k8s version: %s", err) + } + log.Infof("Creating signed kubelet certificate") kubeletCert, err := s.ca.GetCertificate(req.CertificateRequest) if err != nil { @@ -125,9 +132,25 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi DiscoveryTokenCaCertHash: kubeArgs.CACertHashes[0], KubeletCert: kubeletCert, ControlPlaneFiles: controlPlaneFiles, + KubernetesVersion: k8sVersion, }, nil } +// getK8sVersion reads the k8s version from a VolumeMount that is backed by the k8s-version ConfigMap. +func (s *Server) getK8sVersion(ctx context.Context) (string, error) { + fileContent, err := s.file.Read(filepath.Join(constants.ServiceBasePath, "k8s-version")) + if err != nil { + return "", fmt.Errorf("could not read k8s version file: %v", err) + } + k8sVersion := string(fileContent) + + if !versions.IsSupportedK8sVersion(k8sVersion) { + return "", fmt.Errorf("supplied k8s version is not supported: %v", k8sVersion) + } + + return k8sVersion, nil +} + // joinTokenGetter returns Kubernetes bootstrap (join) tokens. type joinTokenGetter interface { // GetJoinToken returns a bootstrap (join) token. diff --git a/joinservice/internal/server/server_test.go b/joinservice/internal/server/server_test.go index cf80ba3e2..0542f54af 100644 --- a/joinservice/internal/server/server_test.go +++ b/joinservice/internal/server/server_test.go @@ -37,6 +37,7 @@ func TestIssueJoinTicket(t *testing.T) { CACertHashes: []string{"hash"}, Token: "token", } + testK8sVersion := "1.23.6" testCases := map[string]struct { isControlPlane bool @@ -115,6 +116,10 @@ func TestIssueJoinTicket(t *testing.T) { if len(tc.id) > 0 { require.NoError(file.Write(filepath.Join(constants.ServiceBasePath, constants.IDFilename), tc.id, 0o644)) } + + // IssueJoinTicket tries to read the k8s-version ConfigMap from a mounted file. + require.NoError(file.Write(filepath.Join(constants.ServiceBasePath, constants.K8sVersion), []byte(testK8sVersion), 0o644)) + api := New( logger.NewTest(t), file, diff --git a/joinservice/joinproto/join.pb.go b/joinservice/joinproto/join.pb.go index 91ce69d44..3782fd06b 100644 --- a/joinservice/joinproto/join.pb.go +++ b/joinservice/joinproto/join.pb.go @@ -96,6 +96,7 @@ type IssueJoinTicketResponse struct { Token string `protobuf:"bytes,6,opt,name=token,proto3" json:"token,omitempty"` DiscoveryTokenCaCertHash string `protobuf:"bytes,7,opt,name=discovery_token_ca_cert_hash,json=discoveryTokenCaCertHash,proto3" json:"discovery_token_ca_cert_hash,omitempty"` ControlPlaneFiles []*ControlPlaneCertOrKey `protobuf:"bytes,8,rep,name=control_plane_files,json=controlPlaneFiles,proto3" json:"control_plane_files,omitempty"` + KubernetesVersion string `protobuf:"bytes,9,opt,name=kubernetes_version,json=kubernetesVersion,proto3" json:"kubernetes_version,omitempty"` } func (x *IssueJoinTicketResponse) Reset() { @@ -186,6 +187,13 @@ func (x *IssueJoinTicketResponse) GetControlPlaneFiles() []*ControlPlaneCertOrKe return nil } +func (x *IssueJoinTicketResponse) GetKubernetesVersion() string { + if x != nil { + return x.KubernetesVersion + } + return "" +} + type ControlPlaneCertOrKey struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -254,7 +262,7 @@ var file_join_proto_rawDesc = []byte{ 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x69, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x69, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x22, 0xf3, 0x02, 0x0a, 0x17, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, + 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x22, 0xa2, 0x03, 0x0a, 0x17, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x73, 0x74, 0x61, 0x74, 0x65, @@ -277,21 +285,24 @@ var file_join_proto_rawDesc = []byte{ 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x6f, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x22, 0x43, 0x0a, 0x19, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x5f, 0x63, 0x65, 0x72, - 0x74, 0x5f, 0x6f, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, - 0x32, 0x55, 0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, 0x4e, 0x0a, 0x0f, 0x49, 0x73, 0x73, 0x75, 0x65, - 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1c, 0x2e, 0x6a, 0x6f, 0x69, - 0x6e, 0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, - 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, - 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, - 0x6a, 0x6f, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x6a, 0x6f, 0x69, 0x6e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x6b, + 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, + 0x74, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x43, 0x0a, 0x19, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x5f, 0x63, 0x65, 0x72, 0x74, + 0x5f, 0x6f, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, + 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x32, + 0x55, 0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, 0x4e, 0x0a, 0x0f, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, + 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1c, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, + 0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x49, + 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, + 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, + 0x6f, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x6a, 0x6f, 0x69, 0x6e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/joinservice/joinproto/join.proto b/joinservice/joinproto/join.proto index 1d26ed387..8c97f4c12 100644 --- a/joinservice/joinproto/join.proto +++ b/joinservice/joinproto/join.proto @@ -24,6 +24,7 @@ message IssueJoinTicketResponse { string token = 6; string discovery_token_ca_cert_hash = 7; repeated control_plane_cert_or_key control_plane_files = 8; + string kubernetes_version = 9; } message control_plane_cert_or_key {