From 52ceced223635125d7721802fa090fdd6fefd647 Mon Sep 17 00:00:00 2001 From: Otto Bittner Date: Fri, 22 Jul 2022 15:05:04 +0200 Subject: [PATCH] AB#2255: Fix kubeadm version incompatibility (#293) * Update image version * Introduce 'ValidK8sVersion' type. Ensures that consumers of the k8sVersion receive a valid version, without having to do their own validation. * Add testcase to check that kubeadm accepts the currently provided version. --- bootstrapper/cloudprovider/azure/ccm.go | 6 +-- bootstrapper/cloudprovider/azure/ccm_test.go | 3 +- .../cloudprovider/azure/cloudnodemanager.go | 9 +--- .../azure/cloudnodemanager_test.go | 3 +- bootstrapper/cloudprovider/gcp/ccm.go | 7 +-- bootstrapper/cloudprovider/gcp/ccm_test.go | 3 +- .../cloudprovider/gcp/cloudnodemanager.go | 4 +- bootstrapper/cloudprovider/qemu/ccm.go | 3 +- .../cloudprovider/qemu/cloudnodemanager.go | 4 +- .../internal/kubernetes/cloud_provider.go | 9 ++-- .../kubernetes/k8sapi/kubeadm_config.go | 6 ++- .../kubernetes/k8sapi/kubeadm_config_test.go | 39 +++++++++++++++- .../k8sapi/resources/cluster_autoscaler.go | 2 +- .../internal/kubernetes/k8sapi/util.go | 8 +--- bootstrapper/internal/kubernetes/k8sutil.go | 3 +- .../internal/kubernetes/kubernetes.go | 36 ++++++++------- .../internal/kubernetes/kubernetes_test.go | 44 +++++++++---------- internal/config/config.go | 2 +- internal/constants/constants.go | 2 - internal/versions/versions.go | 42 +++++++++++++----- joinservice/internal/server/server_test.go | 3 +- 21 files changed, 146 insertions(+), 92 deletions(-) diff --git a/bootstrapper/cloudprovider/azure/ccm.go b/bootstrapper/cloudprovider/azure/ccm.go index 22301cede..f9b1af1c2 100644 --- a/bootstrapper/cloudprovider/azure/ccm.go +++ b/bootstrapper/cloudprovider/azure/ccm.go @@ -3,7 +3,6 @@ package azure import ( "context" "encoding/json" - "fmt" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/internal/azureshared" @@ -30,10 +29,7 @@ func NewCloudControllerManager(metadata ccmMetadata) *CloudControllerManager { } // Image returns the container image used to provide cloud-controller-manager for the cloud-provider. -func (c *CloudControllerManager) Image(k8sVersion string) (string, error) { - if !versions.IsSupportedK8sVersion(k8sVersion) { - return "", fmt.Errorf("received unsupported k8sVersion: %s", k8sVersion) - } +func (c *CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure, nil } diff --git a/bootstrapper/cloudprovider/azure/ccm_test.go b/bootstrapper/cloudprovider/azure/ccm_test.go index bb84c67e6..60eb3947a 100644 --- a/bootstrapper/cloudprovider/azure/ccm_test.go +++ b/bootstrapper/cloudprovider/azure/ccm_test.go @@ -7,6 +7,7 @@ import ( "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/internal/cloud/metadata" + "github.com/edgelesssys/constellation/internal/versions" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" k8s "k8s.io/api/core/v1" @@ -88,7 +89,7 @@ func TestTrivialCCMFunctions(t *testing.T) { assert := assert.New(t) cloud := CloudControllerManager{} - assert.NotEmpty(cloud.Image("1.23")) + assert.NotEmpty(cloud.Image(versions.Latest)) assert.NotEmpty(cloud.Path()) assert.NotEmpty(cloud.Name()) assert.NotEmpty(cloud.ExtraArgs()) diff --git a/bootstrapper/cloudprovider/azure/cloudnodemanager.go b/bootstrapper/cloudprovider/azure/cloudnodemanager.go index b8f67b3d0..44f0d2d78 100644 --- a/bootstrapper/cloudprovider/azure/cloudnodemanager.go +++ b/bootstrapper/cloudprovider/azure/cloudnodemanager.go @@ -1,8 +1,6 @@ package azure import ( - "fmt" - "github.com/edgelesssys/constellation/internal/versions" ) @@ -11,11 +9,8 @@ import ( type CloudNodeManager struct{} // Image returns the container image used to provide cloud-node-manager for the cloud-provider. -func (c *CloudNodeManager) Image(k8sVersion string) (string, error) { - if !versions.IsSupportedK8sVersion(k8sVersion) { - return "", fmt.Errorf("received unsupported k8sVersion: %s", k8sVersion) - } - return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure, nil +func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { + return versions.VersionConfigs[k8sVersion].CloudNodeManagerImageAzure, nil } // Path returns the path used by cloud-node-manager executable within the container image. diff --git a/bootstrapper/cloudprovider/azure/cloudnodemanager_test.go b/bootstrapper/cloudprovider/azure/cloudnodemanager_test.go index d8bfa54bf..76d754388 100644 --- a/bootstrapper/cloudprovider/azure/cloudnodemanager_test.go +++ b/bootstrapper/cloudprovider/azure/cloudnodemanager_test.go @@ -3,6 +3,7 @@ package azure import ( "testing" + "github.com/edgelesssys/constellation/internal/versions" "github.com/stretchr/testify/assert" ) @@ -10,7 +11,7 @@ func TestTrivialCNMFunctions(t *testing.T) { assert := assert.New(t) cloud := CloudNodeManager{} - assert.NotEmpty(cloud.Image("1.23")) + assert.NotEmpty(cloud.Image(versions.Latest)) assert.NotEmpty(cloud.Path()) assert.NotEmpty(cloud.ExtraArgs()) assert.True(cloud.Supported()) diff --git a/bootstrapper/cloudprovider/gcp/ccm.go b/bootstrapper/cloudprovider/gcp/ccm.go index 682c0d2b9..f95e125df 100644 --- a/bootstrapper/cloudprovider/gcp/ccm.go +++ b/bootstrapper/cloudprovider/gcp/ccm.go @@ -18,11 +18,8 @@ import ( type CloudControllerManager struct{} // Image returns the container image used to provide cloud-controller-manager for the cloud-provider. -func (c *CloudControllerManager) Image(k8sVersion string) (string, error) { - if !versions.IsSupportedK8sVersion(k8sVersion) { - return "", fmt.Errorf("received unsupported k8sVersion: %s", k8sVersion) - } - return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure, nil +func (c *CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { + return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageGCP, nil } // Path returns the path used by cloud-controller-manager executable within the container image. diff --git a/bootstrapper/cloudprovider/gcp/ccm_test.go b/bootstrapper/cloudprovider/gcp/ccm_test.go index 12761f6f5..9a7dd2949 100644 --- a/bootstrapper/cloudprovider/gcp/ccm_test.go +++ b/bootstrapper/cloudprovider/gcp/ccm_test.go @@ -8,6 +8,7 @@ import ( "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/edgelesssys/constellation/internal/gcpshared" + "github.com/edgelesssys/constellation/internal/versions" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" k8s "k8s.io/api/core/v1" @@ -132,7 +133,7 @@ func TestTrivialCCMFunctions(t *testing.T) { assert := assert.New(t) cloud := CloudControllerManager{} - assert.NotEmpty(cloud.Image("1.23")) + assert.NotEmpty(cloud.Image(versions.Latest)) assert.NotEmpty(cloud.Path()) assert.NotEmpty(cloud.Name()) assert.NotEmpty(cloud.ExtraArgs()) diff --git a/bootstrapper/cloudprovider/gcp/cloudnodemanager.go b/bootstrapper/cloudprovider/gcp/cloudnodemanager.go index b14e1fcd1..8970ea519 100644 --- a/bootstrapper/cloudprovider/gcp/cloudnodemanager.go +++ b/bootstrapper/cloudprovider/gcp/cloudnodemanager.go @@ -1,11 +1,13 @@ package gcp +import "github.com/edgelesssys/constellation/internal/versions" + // CloudNodeManager holds the GCP cloud-node-manager configuration. type CloudNodeManager struct{} // Image returns the container image used to provide cloud-node-manager for the cloud-provider. // Not used on GCP. -func (c *CloudNodeManager) Image(k8sVersion string) (string, error) { +func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { return "", nil } diff --git a/bootstrapper/cloudprovider/qemu/ccm.go b/bootstrapper/cloudprovider/qemu/ccm.go index 34b4889e6..e689b574c 100644 --- a/bootstrapper/cloudprovider/qemu/ccm.go +++ b/bootstrapper/cloudprovider/qemu/ccm.go @@ -5,6 +5,7 @@ import ( "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/internal/cloud/metadata" + "github.com/edgelesssys/constellation/internal/versions" k8s "k8s.io/api/core/v1" ) @@ -12,7 +13,7 @@ import ( type CloudControllerManager struct{} // Image returns the container image used to provide cloud-controller-manager for the cloud-provider. -func (c CloudControllerManager) Image(k8sVersion string) (string, error) { +func (c CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { return "", nil } diff --git a/bootstrapper/cloudprovider/qemu/cloudnodemanager.go b/bootstrapper/cloudprovider/qemu/cloudnodemanager.go index 49d8c9ebc..44db3a9eb 100644 --- a/bootstrapper/cloudprovider/qemu/cloudnodemanager.go +++ b/bootstrapper/cloudprovider/qemu/cloudnodemanager.go @@ -1,11 +1,13 @@ package qemu +import "github.com/edgelesssys/constellation/internal/versions" + // CloudNodeManager holds the QEMU cloud-node-manager configuration. type CloudNodeManager struct{} // Image returns the container image used to provide cloud-node-manager for the cloud-provider. // Not used on QEMU. -func (c *CloudNodeManager) Image(k8sVersion string) (string, error) { +func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { return "", nil } diff --git a/bootstrapper/internal/kubernetes/cloud_provider.go b/bootstrapper/internal/kubernetes/cloud_provider.go index 33eb241a2..6b03e957a 100644 --- a/bootstrapper/internal/kubernetes/cloud_provider.go +++ b/bootstrapper/internal/kubernetes/cloud_provider.go @@ -5,6 +5,7 @@ import ( "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/internal/cloud/metadata" + "github.com/edgelesssys/constellation/internal/versions" k8s "k8s.io/api/core/v1" ) @@ -29,7 +30,7 @@ type ProviderMetadata interface { // CloudControllerManager implementers provide configuration for the k8s cloud-controller-manager. type CloudControllerManager interface { // Image returns the container image used to provide cloud-controller-manager for the cloud-provider. - Image(k8sVersion string) (string, error) + Image(k8sVersion versions.ValidK8sVersion) (string, error) // Path returns the path used by cloud-controller-manager executable within the container image. Path() string // Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager). @@ -56,7 +57,7 @@ type CloudControllerManager interface { // CloudNodeManager implementers provide configuration for the k8s cloud-node-manager. type CloudNodeManager interface { // Image returns the container image used to provide cloud-node-manager for the cloud-provider. - Image(k8sVersion string) (string, error) + Image(k8sVersion versions.ValidK8sVersion) (string, error) // Path returns the path used by cloud-node-manager executable within the container image. Path() string // ExtraArgs returns a list of arguments to append to the cloud-node-manager command. @@ -133,7 +134,7 @@ type stubCloudControllerManager struct { SupportedResp bool } -func (m *stubCloudControllerManager) Image(k8sVersion string) (string, error) { +func (m *stubCloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { return "stub-image:latest", nil } @@ -181,7 +182,7 @@ type stubCloudNodeManager struct { ExtraArgsResp []string } -func (m *stubCloudNodeManager) Image(k8sVersion string) (string, error) { +func (m *stubCloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) { return m.ImageResp, nil } diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go index 1a23f1a5d..97134fe2f 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go @@ -5,6 +5,7 @@ import ( "github.com/edgelesssys/constellation/bootstrapper/internal/kubelet" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" + "github.com/edgelesssys/constellation/internal/versions" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconf "k8s.io/kubelet/config/v1beta1" @@ -24,11 +25,12 @@ const ( type CoreOSConfiguration struct{} -func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sVersion string) KubeadmInitYAML { +func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sVersion versions.ValidK8sVersion) KubeadmInitYAML { var cloudProvider string if externalCloudProvider { cloudProvider = "external" } + return KubeadmInitYAML{ InitConfiguration: kubeadm.InitConfiguration{ TypeMeta: metav1.TypeMeta{ @@ -54,7 +56,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sV APIVersion: kubeadm.SchemeGroupVersion.String(), }, // Target kubernetes version of the control plane. - KubernetesVersion: k8sVersion, + KubernetesVersion: versions.VersionConfigs[k8sVersion].PatchVersion, // necessary to be able to access the kubeapi server through localhost APIServer: kubeadm.APIServer{ ControlPlaneComponent: kubeadm.ControlPlaneComponent{ diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go index 28538fabe..79098ce15 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go @@ -1,11 +1,14 @@ package k8sapi import ( + "fmt" "testing" + "github.com/edgelesssys/constellation/internal/versions" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" + kubeadmUtil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) func TestMain(m *testing.M) { @@ -19,11 +22,11 @@ func TestInitConfiguration(t *testing.T) { config KubeadmInitYAML }{ "CoreOS init config can be created": { - config: coreOSConfig.InitConfiguration(true, "3.2.1"), + config: coreOSConfig.InitConfiguration(true, versions.Latest), }, "CoreOS init config with all fields can be created": { config: func() KubeadmInitYAML { - c := coreOSConfig.InitConfiguration(true, "3.2.1") + c := coreOSConfig.InitConfiguration(true, versions.Latest) c.SetAPIServerAdvertiseAddress("192.0.2.0") c.SetNodeIP("192.0.2.0") c.SetNodeName("node") @@ -50,6 +53,38 @@ func TestInitConfiguration(t *testing.T) { } } +func TestInitConfigurationKubeadmCompatibility(t *testing.T) { + coreOSConfig := CoreOSConfiguration{} + + testCases := map[string]struct { + config KubeadmInitYAML + expectedVersion string + wantErr bool + }{ + "Kubeadm accepts version 'Latest'": { + config: coreOSConfig.InitConfiguration(true, versions.Latest), + expectedVersion: fmt.Sprintf("v%s", versions.VersionConfigs[versions.Latest].PatchVersion), + }, + "Kubeadm receives incompatible version": { + config: coreOSConfig.InitConfiguration(true, "1.22"), + wantErr: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + version, err := kubeadmUtil.KubernetesReleaseVersion(tc.config.ClusterConfiguration.KubernetesVersion) + if tc.wantErr { + assert.Error(err) + return + } + assert.Equal(tc.expectedVersion, version) + assert.NoError(err) + }) + } +} + func TestJoinConfiguration(t *testing.T) { coreOSConfig := CoreOSConfiguration{} diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go b/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go index f5079d35e..778e9643e 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go +++ b/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go @@ -23,7 +23,7 @@ type autoscalerDeployment struct { } // NewDefaultAutoscalerDeployment creates a new *autoscalerDeployment, customized for the CSP. -func NewDefaultAutoscalerDeployment(extraVolumes []k8s.Volume, extraVolumeMounts []k8s.VolumeMount, env []k8s.EnvVar, k8sVersion string) *autoscalerDeployment { +func NewDefaultAutoscalerDeployment(extraVolumes []k8s.Volume, extraVolumeMounts []k8s.VolumeMount, env []k8s.EnvVar, k8sVersion versions.ValidK8sVersion) *autoscalerDeployment { return &autoscalerDeployment{ PodDisruptionBudget: policy.PodDisruptionBudget{ TypeMeta: v1.TypeMeta{ diff --git a/bootstrapper/internal/kubernetes/k8sapi/util.go b/bootstrapper/internal/kubernetes/k8sapi/util.go index 0f4f5f0cd..1ed1a7fb8 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/util.go +++ b/bootstrapper/internal/kubernetes/k8sapi/util.go @@ -70,12 +70,8 @@ func NewKubernetesUtil() *KubernetesUtil { } // InstallComponents installs kubernetes components in the version specified. -func (k *KubernetesUtil) InstallComponents(ctx context.Context, version string) error { - var versionConf versions.KubernetesVersion - var ok bool - if versionConf, ok = versions.VersionConfigs[version]; !ok { - return fmt.Errorf("unsupported kubernetes version %q", version) - } +func (k *KubernetesUtil) InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error { + versionConf := versions.VersionConfigs[version] if err := k.inst.Install( ctx, versionConf.CNIPluginsURL, []string{cniPluginsDir}, executablePerm, true, diff --git a/bootstrapper/internal/kubernetes/k8sutil.go b/bootstrapper/internal/kubernetes/k8sutil.go index de27df9d9..43053a8f2 100644 --- a/bootstrapper/internal/kubernetes/k8sutil.go +++ b/bootstrapper/internal/kubernetes/k8sutil.go @@ -7,10 +7,11 @@ import ( "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/internal/logger" + "github.com/edgelesssys/constellation/internal/versions" ) type clusterUtil interface { - InstallComponents(ctx context.Context, version string) error + InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, log *logger.Logger) error JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput) error diff --git a/bootstrapper/internal/kubernetes/kubernetes.go b/bootstrapper/internal/kubernetes/kubernetes.go index 5bb2d4e12..251fb1b79 100644 --- a/bootstrapper/internal/kubernetes/kubernetes.go +++ b/bootstrapper/internal/kubernetes/kubernetes.go @@ -32,7 +32,7 @@ type configReader interface { // configurationProvider provides kubeadm init and join configuration. type configurationProvider interface { - InitConfiguration(externalCloudProvider bool, k8sVersion string) k8sapi.KubeadmInitYAML + InitConfiguration(externalCloudProvider bool, k8sVersion versions.ValidK8sVersion) k8sapi.KubeadmInitYAML JoinConfiguration(externalCloudProvider bool) k8sapi.KubeadmJoinYAML } @@ -80,12 +80,14 @@ type KMSConfig struct { // InitCluster initializes a new Kubernetes cluster and applies pod network provider. func (k *KubeWrapper) InitCluster( - ctx context.Context, autoscalingNodeGroups []string, cloudServiceAccountURI, k8sVersion string, + ctx context.Context, autoscalingNodeGroups []string, cloudServiceAccountURI, versionString string, id attestationtypes.ID, kmsConfig KMSConfig, sshUsers map[string]string, log *logger.Logger, ) ([]byte, error) { - log.With(zap.String("version", k8sVersion)).Infof("Installing Kubernetes components") - // InstallComponents validates the k8sVersion as it's first action and returns if not supported. - // This implicitly makes k8sVersion safe to use in this function. + k8sVersion, err := versions.NewValidK8sVersion(versionString) + if err != nil { + return nil, err + } + log.With(zap.String("version", string(k8sVersion))).Infof("Installing Kubernetes components") if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil { return nil, err } @@ -236,8 +238,12 @@ func (k *KubeWrapper) InitCluster( } // JoinCluster joins existing Kubernetes cluster. -func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, k8sVersion string, log *logger.Logger) error { - log.With(zap.String("version", k8sVersion)).Infof("Installing Kubernetes components") +func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, versionString string, log *logger.Logger) error { + k8sVersion, err := versions.NewValidK8sVersion(versionString) + if err != nil { + return err + } + log.With(zap.String("version", string(k8sVersion))).Infof("Installing Kubernetes components") if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil { return err } @@ -311,7 +317,7 @@ func (k *KubeWrapper) setupJoinService(csp string, measurementsJSON []byte, id a return k.clusterUtil.SetupJoinService(k.client, joinConfiguration) } -func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServiceAccountURI string, instance metadata.InstanceMetadata, k8sVersion string) error { +func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServiceAccountURI string, instance metadata.InstanceMetadata, k8sVersion versions.ValidK8sVersion) error { if !k.cloudControllerManager.Supported() { return nil } @@ -339,7 +345,7 @@ func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServ return nil } -func (k *KubeWrapper) setupCloudNodeManager(k8sVersion string) error { +func (k *KubeWrapper) setupCloudNodeManager(k8sVersion versions.ValidK8sVersion) error { if !k.cloudNodeManager.Supported() { return nil } @@ -358,7 +364,7 @@ func (k *KubeWrapper) setupCloudNodeManager(k8sVersion string) error { return nil } -func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata, cloudServiceAccountURI string, autoscalingNodeGroups []string, k8sVersion string) error { +func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata, cloudServiceAccountURI string, autoscalingNodeGroups []string, k8sVersion versions.ValidK8sVersion) error { if !k.clusterAutoscaler.Supported() { return nil } @@ -377,11 +383,7 @@ func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata, } // setupK8sVersionConfigMap applies a ConfigMap (cf. server-side apply) to consistently store the installed k8s version. -func (k *KubeWrapper) setupK8sVersionConfigMap(ctx context.Context, k8sVersion string) error { - if !versions.IsSupportedK8sVersion(k8sVersion) { - return fmt.Errorf("supplied k8s version is not supported: %v", k8sVersion) - } - +func (k *KubeWrapper) setupK8sVersionConfigMap(ctx context.Context, k8sVersion versions.ValidK8sVersion) error { config := corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", @@ -392,14 +394,14 @@ func (k *KubeWrapper) setupK8sVersionConfigMap(ctx context.Context, k8sVersion s Namespace: "kube-system", }, Data: map[string]string{ - constants.K8sVersion: k8sVersion, + constants.K8sVersion: string(k8sVersion), }, } // We do not use the client's Apply method here since we are handling a kubernetes-native type. // These types don't implement our custom Marshaler interface. if err := k.client.CreateConfigMap(ctx, config); err != nil { - return fmt.Errorf("Apply in KubeWrapper.setupK8sVersionConfigMap(..) failed with: %v", err) + return fmt.Errorf("apply in KubeWrapper.setupK8sVersionConfigMap(..) failed with: %v", err) } return nil diff --git a/bootstrapper/internal/kubernetes/kubernetes_test.go b/bootstrapper/internal/kubernetes/kubernetes_test.go index f2c06a710..af1135091 100644 --- a/bootstrapper/internal/kubernetes/kubernetes_test.go +++ b/bootstrapper/internal/kubernetes/kubernetes_test.go @@ -13,6 +13,7 @@ import ( attestationtypes "github.com/edgelesssys/constellation/internal/attestation/types" "github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/edgelesssys/constellation/internal/logger" + "github.com/edgelesssys/constellation/internal/versions" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" @@ -47,7 +48,7 @@ func TestInitCluster(t *testing.T) { kubeconfigReader configReader wantConfig k8sapi.KubeadmInitYAML wantErr bool - k8sVersion string + k8sVersion versions.ValidK8sVersion }{ "kubeadm init works without metadata": { clusterUtil: stubClusterUtil{}, @@ -70,7 +71,7 @@ func TestInitCluster(t *testing.T) { }, ClusterConfiguration: kubeadm.ClusterConfiguration{}, }, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init works with metadata and loadbalancer": { clusterUtil: stubClusterUtil{}, @@ -110,7 +111,7 @@ func TestInitCluster(t *testing.T) { }, }, wantErr: false, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init fails when retrieving metadata self": { clusterUtil: stubClusterUtil{}, @@ -125,7 +126,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init fails when retrieving metadata subnetwork cidr": { clusterUtil: stubClusterUtil{}, @@ -140,7 +141,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init fails when retrieving metadata loadbalancer ip": { clusterUtil: stubClusterUtil{}, @@ -156,7 +157,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init fails when applying the init config": { clusterUtil: stubClusterUtil{initClusterErr: someErr}, @@ -168,7 +169,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init fails when setting up the pod network": { clusterUtil: stubClusterUtil{setupPodNetworkErr: someErr}, @@ -180,7 +181,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init fails when setting up the join service": { clusterUtil: stubClusterUtil{setupJoinServiceError: someErr}, @@ -192,7 +193,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init fails when setting the cloud contoller manager": { clusterUtil: stubClusterUtil{setupCloudControllerManagerError: someErr}, @@ -204,7 +205,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init fails when setting the cloud node manager": { clusterUtil: stubClusterUtil{setupCloudNodeManagerError: someErr}, @@ -216,7 +217,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{SupportedResp: true}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init fails when setting the cluster autoscaler": { clusterUtil: stubClusterUtil{setupAutoscalingError: someErr}, @@ -228,7 +229,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{SupportedResp: true}, wantErr: true, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init fails when reading kubeconfig": { clusterUtil: stubClusterUtil{}, @@ -240,7 +241,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init fails when setting up the kms": { clusterUtil: stubClusterUtil{setupKMSError: someErr}, @@ -252,7 +253,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{SupportedResp: false}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "kubeadm init fails when setting up verification service": { clusterUtil: stubClusterUtil{setupVerificationServiceErr: someErr}, @@ -264,7 +265,7 @@ func TestInitCluster(t *testing.T) { CloudNodeManager: &stubCloudNodeManager{SupportedResp: false}, ClusterAutoscaler: &stubClusterAutoscaler{}, wantErr: true, - k8sVersion: "1.23", + k8sVersion: versions.Latest, }, "unsupported k8sVersion fails cluster creation": { clusterUtil: stubClusterUtil{}, @@ -275,7 +276,7 @@ func TestInitCluster(t *testing.T) { CloudControllerManager: &stubCloudControllerManager{}, CloudNodeManager: &stubCloudNodeManager{}, ClusterAutoscaler: &stubClusterAutoscaler{}, - k8sVersion: "invalid version", + k8sVersion: "1.19", wantErr: true, }, } @@ -296,7 +297,7 @@ func TestInitCluster(t *testing.T) { kubeconfigReader: tc.kubeconfigReader, getIPAddr: func() (string, error) { return privateIP, nil }, } - _, err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountURI, tc.k8sVersion, attestationtypes.ID{}, KMSConfig{MasterSecret: masterSecret}, nil, logger.NewTest(t)) + _, err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountURI, string(tc.k8sVersion), attestationtypes.ID{}, KMSConfig{MasterSecret: masterSecret}, nil, logger.NewTest(t)) if tc.wantErr { assert.Error(err) @@ -321,8 +322,7 @@ func TestJoinCluster(t *testing.T) { } privateIP := "192.0.2.1" - // stubClusterUtil does not validate the k8sVersion, thus it can be arbitrary. - k8sVersion := "3.2.1" + k8sVersion := versions.Latest testCases := map[string]struct { clusterUtil stubClusterUtil @@ -454,7 +454,7 @@ func TestJoinCluster(t *testing.T) { getIPAddr: func() (string, error) { return privateIP, nil }, } - err := kube.JoinCluster(context.Background(), joinCommand, tc.role, k8sVersion, logger.NewTest(t)) + err := kube.JoinCluster(context.Background(), joinCommand, tc.role, string(k8sVersion), logger.NewTest(t)) if tc.wantErr { assert.Error(err) return @@ -522,7 +522,7 @@ type stubClusterUtil struct { joinConfigs [][]byte } -func (s *stubClusterUtil) InstallComponents(ctx context.Context, version string) error { +func (s *stubClusterUtil) InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error { return s.installComponentsErr } @@ -588,7 +588,7 @@ type stubConfigProvider struct { JoinConfig k8sapi.KubeadmJoinYAML } -func (s *stubConfigProvider) InitConfiguration(_ bool, _ string) k8sapi.KubeadmInitYAML { +func (s *stubConfigProvider) InitConfiguration(_ bool, _ versions.ValidK8sVersion) k8sapi.KubeadmInitYAML { return s.InitConfig } diff --git a/internal/config/config.go b/internal/config/config.go index 28580a6f3..69b1a16d4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -230,7 +230,7 @@ func Default() *Config { Measurements: qemuPCRs, }, }, - KubernetesVersion: "1.23", + KubernetesVersion: string(versions.Latest), } } diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 8432188c0..dcfe4e37e 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -97,8 +97,6 @@ const ( // Kubernetes. // - // KubernetesVersion installed by kubeadm. - KubernetesVersion = "stable-1.23" KubernetesJoinTokenTTL = 15 * time.Minute ) diff --git a/internal/versions/versions.go b/internal/versions/versions.go index 8cd66f6b6..bc1cea2e3 100644 --- a/internal/versions/versions.go +++ b/internal/versions/versions.go @@ -1,5 +1,28 @@ package versions +import "fmt" + +// ValidK8sVersion represents any of the three currently supported k8s versions. +type ValidK8sVersion string + +// NewValidK8sVersion validates the given string and produces a new ValidK8sVersion object. +func NewValidK8sVersion(k8sVersion string) (ValidK8sVersion, error) { + if IsSupportedK8sVersion(k8sVersion) { + return ValidK8sVersion(k8sVersion), nil + } + return "", fmt.Errorf("invalid k8sVersion supplied: %s", k8sVersion) +} + +// IsSupportedK8sVersion checks if a given Kubernetes version is supported by Constellation. +func IsSupportedK8sVersion(version string) bool { + switch version { + case string(V1_23): + return true + default: + return false + } +} + const ( // Constellation images. // These images are built in a way that they support all versions currently listed in VersionConfigs. @@ -8,11 +31,17 @@ const ( KmsImage = "ghcr.io/edgelesssys/constellation/kmsserver:v1.3.2-0.20220714151638-d295be31" VerificationImage = "ghcr.io/edgelesssys/constellation/verification-service:v1.3.2-0.20220714151638-d295be31" GcpGuestImage = "ghcr.io/edgelesssys/gcp-guest-agent:latest" + + // currently supported versions. + V1_23 ValidK8sVersion = "1.23" + Latest ValidK8sVersion = V1_23 ) // versionConfigs holds download URLs for all required kubernetes components for every supported version. -var VersionConfigs map[string]KubernetesVersion = map[string]KubernetesVersion{ - "1.23": { +var VersionConfigs map[ValidK8sVersion]KubernetesVersion = map[ValidK8sVersion]KubernetesVersion{ + // TODO: + V1_23: { + PatchVersion: "1.23.6", CNIPluginsURL: "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz", CrictlURL: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.1/crictl-v1.24.1-linux-amd64.tar.gz", KubeletServiceURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service", @@ -20,8 +49,6 @@ var VersionConfigs map[string]KubernetesVersion = map[string]KubernetesVersion{ KubeletURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubelet", KubeadmURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubeadm", KubectlURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl", - // CloudControllerManagerImageAWS is the CCM image used on AWS. - CloudControllerManagerImageAWS: "us.gcr.io/k8s-artifacts-prod/provider-aws/cloud-controller-manager:v1.22.0-alpha.0", // CloudControllerManagerImageGCP is the CCM image used on GCP. // TODO: use newer "cloud-provider-gcp" from https://github.com/kubernetes/cloud-provider-gcp when newer releases are available. CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:sha-2f6a5b07fc2d37f24f8ff725132f87584d627d8f", @@ -36,6 +63,7 @@ var VersionConfigs map[string]KubernetesVersion = map[string]KubernetesVersion{ // KubernetesVersion bundles download URLs to all version-releated binaries necessary for installing/deploying a particular Kubernetes version. type KubernetesVersion struct { + PatchVersion string CNIPluginsURL string CrictlURL string KubeletServiceURL string @@ -49,9 +77,3 @@ type KubernetesVersion struct { CloudNodeManagerImageAzure string ClusterAutoscalerImage string } - -// IsSupportedK8sVersion checks if a given Kubernetes version is supported by Constellation. -func IsSupportedK8sVersion(version string) bool { - _, ok := VersionConfigs[version] - return ok -} diff --git a/joinservice/internal/server/server_test.go b/joinservice/internal/server/server_test.go index 55a213732..078c2650c 100644 --- a/joinservice/internal/server/server_test.go +++ b/joinservice/internal/server/server_test.go @@ -12,6 +12,7 @@ import ( "github.com/edgelesssys/constellation/internal/constants" "github.com/edgelesssys/constellation/internal/file" "github.com/edgelesssys/constellation/internal/logger" + "github.com/edgelesssys/constellation/internal/versions" "github.com/edgelesssys/constellation/joinservice/joinproto" "github.com/spf13/afero" "github.com/stretchr/testify/assert" @@ -37,7 +38,7 @@ func TestIssueJoinTicket(t *testing.T) { CACertHashes: []string{"hash"}, Token: "token", } - testK8sVersion := "1.23" + testK8sVersion := versions.Latest testCases := map[string]struct { isControlPlane bool