AB#2255: Fix kubeadm version incompatibility (#293)

* Update image version
* Introduce 'ValidK8sVersion' type. Ensures that consumers
of the k8sVersion receive a valid version, without
having to do their own validation.
* Add testcase to check that kubeadm accepts the currently provided
version.
This commit is contained in:
Otto Bittner 2022-07-22 15:05:04 +02:00 committed by GitHub
parent ebf76ae7e3
commit 52ceced223
21 changed files with 146 additions and 92 deletions

View File

@ -3,7 +3,6 @@ package azure
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"fmt"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/azureshared" "github.com/edgelesssys/constellation/internal/azureshared"
@ -30,10 +29,7 @@ func NewCloudControllerManager(metadata ccmMetadata) *CloudControllerManager {
} }
// Image returns the container image used to provide cloud-controller-manager for the cloud-provider. // Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
func (c *CloudControllerManager) Image(k8sVersion string) (string, error) { func (c *CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
if !versions.IsSupportedK8sVersion(k8sVersion) {
return "", fmt.Errorf("received unsupported k8sVersion: %s", k8sVersion)
}
return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure, nil return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure, nil
} }

View File

@ -7,6 +7,7 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
k8s "k8s.io/api/core/v1" k8s "k8s.io/api/core/v1"
@ -88,7 +89,7 @@ func TestTrivialCCMFunctions(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
cloud := CloudControllerManager{} cloud := CloudControllerManager{}
assert.NotEmpty(cloud.Image("1.23")) assert.NotEmpty(cloud.Image(versions.Latest))
assert.NotEmpty(cloud.Path()) assert.NotEmpty(cloud.Path())
assert.NotEmpty(cloud.Name()) assert.NotEmpty(cloud.Name())
assert.NotEmpty(cloud.ExtraArgs()) assert.NotEmpty(cloud.ExtraArgs())

View File

@ -1,8 +1,6 @@
package azure package azure
import ( import (
"fmt"
"github.com/edgelesssys/constellation/internal/versions" "github.com/edgelesssys/constellation/internal/versions"
) )
@ -11,11 +9,8 @@ import (
type CloudNodeManager struct{} type CloudNodeManager struct{}
// Image returns the container image used to provide cloud-node-manager for the cloud-provider. // Image returns the container image used to provide cloud-node-manager for the cloud-provider.
func (c *CloudNodeManager) Image(k8sVersion string) (string, error) { func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
if !versions.IsSupportedK8sVersion(k8sVersion) { return versions.VersionConfigs[k8sVersion].CloudNodeManagerImageAzure, nil
return "", fmt.Errorf("received unsupported k8sVersion: %s", k8sVersion)
}
return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure, nil
} }
// Path returns the path used by cloud-node-manager executable within the container image. // Path returns the path used by cloud-node-manager executable within the container image.

View File

@ -3,6 +3,7 @@ package azure
import ( import (
"testing" "testing"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
@ -10,7 +11,7 @@ func TestTrivialCNMFunctions(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
cloud := CloudNodeManager{} cloud := CloudNodeManager{}
assert.NotEmpty(cloud.Image("1.23")) assert.NotEmpty(cloud.Image(versions.Latest))
assert.NotEmpty(cloud.Path()) assert.NotEmpty(cloud.Path())
assert.NotEmpty(cloud.ExtraArgs()) assert.NotEmpty(cloud.ExtraArgs())
assert.True(cloud.Supported()) assert.True(cloud.Supported())

View File

@ -18,11 +18,8 @@ import (
type CloudControllerManager struct{} type CloudControllerManager struct{}
// Image returns the container image used to provide cloud-controller-manager for the cloud-provider. // Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
func (c *CloudControllerManager) Image(k8sVersion string) (string, error) { func (c *CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
if !versions.IsSupportedK8sVersion(k8sVersion) { return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageGCP, nil
return "", fmt.Errorf("received unsupported k8sVersion: %s", k8sVersion)
}
return versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure, nil
} }
// Path returns the path used by cloud-controller-manager executable within the container image. // Path returns the path used by cloud-controller-manager executable within the container image.

View File

@ -8,6 +8,7 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/gcpshared" "github.com/edgelesssys/constellation/internal/gcpshared"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
k8s "k8s.io/api/core/v1" k8s "k8s.io/api/core/v1"
@ -132,7 +133,7 @@ func TestTrivialCCMFunctions(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
cloud := CloudControllerManager{} cloud := CloudControllerManager{}
assert.NotEmpty(cloud.Image("1.23")) assert.NotEmpty(cloud.Image(versions.Latest))
assert.NotEmpty(cloud.Path()) assert.NotEmpty(cloud.Path())
assert.NotEmpty(cloud.Name()) assert.NotEmpty(cloud.Name())
assert.NotEmpty(cloud.ExtraArgs()) assert.NotEmpty(cloud.ExtraArgs())

View File

@ -1,11 +1,13 @@
package gcp package gcp
import "github.com/edgelesssys/constellation/internal/versions"
// CloudNodeManager holds the GCP cloud-node-manager configuration. // CloudNodeManager holds the GCP cloud-node-manager configuration.
type CloudNodeManager struct{} type CloudNodeManager struct{}
// Image returns the container image used to provide cloud-node-manager for the cloud-provider. // Image returns the container image used to provide cloud-node-manager for the cloud-provider.
// Not used on GCP. // Not used on GCP.
func (c *CloudNodeManager) Image(k8sVersion string) (string, error) { func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return "", nil return "", nil
} }

View File

@ -5,6 +5,7 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/versions"
k8s "k8s.io/api/core/v1" k8s "k8s.io/api/core/v1"
) )
@ -12,7 +13,7 @@ import (
type CloudControllerManager struct{} type CloudControllerManager struct{}
// Image returns the container image used to provide cloud-controller-manager for the cloud-provider. // Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
func (c CloudControllerManager) Image(k8sVersion string) (string, error) { func (c CloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return "", nil return "", nil
} }

View File

@ -1,11 +1,13 @@
package qemu package qemu
import "github.com/edgelesssys/constellation/internal/versions"
// CloudNodeManager holds the QEMU cloud-node-manager configuration. // CloudNodeManager holds the QEMU cloud-node-manager configuration.
type CloudNodeManager struct{} type CloudNodeManager struct{}
// Image returns the container image used to provide cloud-node-manager for the cloud-provider. // Image returns the container image used to provide cloud-node-manager for the cloud-provider.
// Not used on QEMU. // Not used on QEMU.
func (c *CloudNodeManager) Image(k8sVersion string) (string, error) { func (c *CloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return "", nil return "", nil
} }

View File

@ -5,6 +5,7 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/versions"
k8s "k8s.io/api/core/v1" k8s "k8s.io/api/core/v1"
) )
@ -29,7 +30,7 @@ type ProviderMetadata interface {
// CloudControllerManager implementers provide configuration for the k8s cloud-controller-manager. // CloudControllerManager implementers provide configuration for the k8s cloud-controller-manager.
type CloudControllerManager interface { type CloudControllerManager interface {
// Image returns the container image used to provide cloud-controller-manager for the cloud-provider. // Image returns the container image used to provide cloud-controller-manager for the cloud-provider.
Image(k8sVersion string) (string, error) Image(k8sVersion versions.ValidK8sVersion) (string, error)
// Path returns the path used by cloud-controller-manager executable within the container image. // Path returns the path used by cloud-controller-manager executable within the container image.
Path() string Path() string
// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager). // Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager).
@ -56,7 +57,7 @@ type CloudControllerManager interface {
// CloudNodeManager implementers provide configuration for the k8s cloud-node-manager. // CloudNodeManager implementers provide configuration for the k8s cloud-node-manager.
type CloudNodeManager interface { type CloudNodeManager interface {
// Image returns the container image used to provide cloud-node-manager for the cloud-provider. // Image returns the container image used to provide cloud-node-manager for the cloud-provider.
Image(k8sVersion string) (string, error) Image(k8sVersion versions.ValidK8sVersion) (string, error)
// Path returns the path used by cloud-node-manager executable within the container image. // Path returns the path used by cloud-node-manager executable within the container image.
Path() string Path() string
// ExtraArgs returns a list of arguments to append to the cloud-node-manager command. // ExtraArgs returns a list of arguments to append to the cloud-node-manager command.
@ -133,7 +134,7 @@ type stubCloudControllerManager struct {
SupportedResp bool SupportedResp bool
} }
func (m *stubCloudControllerManager) Image(k8sVersion string) (string, error) { func (m *stubCloudControllerManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return "stub-image:latest", nil return "stub-image:latest", nil
} }
@ -181,7 +182,7 @@ type stubCloudNodeManager struct {
ExtraArgsResp []string ExtraArgsResp []string
} }
func (m *stubCloudNodeManager) Image(k8sVersion string) (string, error) { func (m *stubCloudNodeManager) Image(k8sVersion versions.ValidK8sVersion) (string, error) {
return m.ImageResp, nil return m.ImageResp, nil
} }

View File

@ -5,6 +5,7 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubelet" "github.com/edgelesssys/constellation/bootstrapper/internal/kubelet"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/versions"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletconf "k8s.io/kubelet/config/v1beta1" kubeletconf "k8s.io/kubelet/config/v1beta1"
@ -24,11 +25,12 @@ const (
type CoreOSConfiguration struct{} type CoreOSConfiguration struct{}
func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sVersion string) KubeadmInitYAML { func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sVersion versions.ValidK8sVersion) KubeadmInitYAML {
var cloudProvider string var cloudProvider string
if externalCloudProvider { if externalCloudProvider {
cloudProvider = "external" cloudProvider = "external"
} }
return KubeadmInitYAML{ return KubeadmInitYAML{
InitConfiguration: kubeadm.InitConfiguration{ InitConfiguration: kubeadm.InitConfiguration{
TypeMeta: metav1.TypeMeta{ TypeMeta: metav1.TypeMeta{
@ -54,7 +56,7 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sV
APIVersion: kubeadm.SchemeGroupVersion.String(), APIVersion: kubeadm.SchemeGroupVersion.String(),
}, },
// Target kubernetes version of the control plane. // Target kubernetes version of the control plane.
KubernetesVersion: k8sVersion, KubernetesVersion: versions.VersionConfigs[k8sVersion].PatchVersion,
// necessary to be able to access the kubeapi server through localhost // necessary to be able to access the kubeapi server through localhost
APIServer: kubeadm.APIServer{ APIServer: kubeadm.APIServer{
ControlPlaneComponent: kubeadm.ControlPlaneComponent{ ControlPlaneComponent: kubeadm.ControlPlaneComponent{

View File

@ -1,11 +1,14 @@
package k8sapi package k8sapi
import ( import (
"fmt"
"testing" "testing"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.uber.org/goleak" "go.uber.org/goleak"
kubeadmUtil "k8s.io/kubernetes/cmd/kubeadm/app/util"
) )
func TestMain(m *testing.M) { func TestMain(m *testing.M) {
@ -19,11 +22,11 @@ func TestInitConfiguration(t *testing.T) {
config KubeadmInitYAML config KubeadmInitYAML
}{ }{
"CoreOS init config can be created": { "CoreOS init config can be created": {
config: coreOSConfig.InitConfiguration(true, "3.2.1"), config: coreOSConfig.InitConfiguration(true, versions.Latest),
}, },
"CoreOS init config with all fields can be created": { "CoreOS init config with all fields can be created": {
config: func() KubeadmInitYAML { config: func() KubeadmInitYAML {
c := coreOSConfig.InitConfiguration(true, "3.2.1") c := coreOSConfig.InitConfiguration(true, versions.Latest)
c.SetAPIServerAdvertiseAddress("192.0.2.0") c.SetAPIServerAdvertiseAddress("192.0.2.0")
c.SetNodeIP("192.0.2.0") c.SetNodeIP("192.0.2.0")
c.SetNodeName("node") c.SetNodeName("node")
@ -50,6 +53,38 @@ func TestInitConfiguration(t *testing.T) {
} }
} }
func TestInitConfigurationKubeadmCompatibility(t *testing.T) {
coreOSConfig := CoreOSConfiguration{}
testCases := map[string]struct {
config KubeadmInitYAML
expectedVersion string
wantErr bool
}{
"Kubeadm accepts version 'Latest'": {
config: coreOSConfig.InitConfiguration(true, versions.Latest),
expectedVersion: fmt.Sprintf("v%s", versions.VersionConfigs[versions.Latest].PatchVersion),
},
"Kubeadm receives incompatible version": {
config: coreOSConfig.InitConfiguration(true, "1.22"),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
version, err := kubeadmUtil.KubernetesReleaseVersion(tc.config.ClusterConfiguration.KubernetesVersion)
if tc.wantErr {
assert.Error(err)
return
}
assert.Equal(tc.expectedVersion, version)
assert.NoError(err)
})
}
}
func TestJoinConfiguration(t *testing.T) { func TestJoinConfiguration(t *testing.T) {
coreOSConfig := CoreOSConfiguration{} coreOSConfig := CoreOSConfiguration{}

View File

@ -23,7 +23,7 @@ type autoscalerDeployment struct {
} }
// NewDefaultAutoscalerDeployment creates a new *autoscalerDeployment, customized for the CSP. // NewDefaultAutoscalerDeployment creates a new *autoscalerDeployment, customized for the CSP.
func NewDefaultAutoscalerDeployment(extraVolumes []k8s.Volume, extraVolumeMounts []k8s.VolumeMount, env []k8s.EnvVar, k8sVersion string) *autoscalerDeployment { func NewDefaultAutoscalerDeployment(extraVolumes []k8s.Volume, extraVolumeMounts []k8s.VolumeMount, env []k8s.EnvVar, k8sVersion versions.ValidK8sVersion) *autoscalerDeployment {
return &autoscalerDeployment{ return &autoscalerDeployment{
PodDisruptionBudget: policy.PodDisruptionBudget{ PodDisruptionBudget: policy.PodDisruptionBudget{
TypeMeta: v1.TypeMeta{ TypeMeta: v1.TypeMeta{

View File

@ -70,12 +70,8 @@ func NewKubernetesUtil() *KubernetesUtil {
} }
// InstallComponents installs kubernetes components in the version specified. // InstallComponents installs kubernetes components in the version specified.
func (k *KubernetesUtil) InstallComponents(ctx context.Context, version string) error { func (k *KubernetesUtil) InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error {
var versionConf versions.KubernetesVersion versionConf := versions.VersionConfigs[version]
var ok bool
if versionConf, ok = versions.VersionConfigs[version]; !ok {
return fmt.Errorf("unsupported kubernetes version %q", version)
}
if err := k.inst.Install( if err := k.inst.Install(
ctx, versionConf.CNIPluginsURL, []string{cniPluginsDir}, executablePerm, true, ctx, versionConf.CNIPluginsURL, []string{cniPluginsDir}, executablePerm, true,

View File

@ -7,10 +7,11 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/logger" "github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/versions"
) )
type clusterUtil interface { type clusterUtil interface {
InstallComponents(ctx context.Context, version string) error InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error
InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, log *logger.Logger) error InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, log *logger.Logger) error
JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error
SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput) error SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput) error

View File

@ -32,7 +32,7 @@ type configReader interface {
// configurationProvider provides kubeadm init and join configuration. // configurationProvider provides kubeadm init and join configuration.
type configurationProvider interface { type configurationProvider interface {
InitConfiguration(externalCloudProvider bool, k8sVersion string) k8sapi.KubeadmInitYAML InitConfiguration(externalCloudProvider bool, k8sVersion versions.ValidK8sVersion) k8sapi.KubeadmInitYAML
JoinConfiguration(externalCloudProvider bool) k8sapi.KubeadmJoinYAML JoinConfiguration(externalCloudProvider bool) k8sapi.KubeadmJoinYAML
} }
@ -80,12 +80,14 @@ type KMSConfig struct {
// InitCluster initializes a new Kubernetes cluster and applies pod network provider. // InitCluster initializes a new Kubernetes cluster and applies pod network provider.
func (k *KubeWrapper) InitCluster( func (k *KubeWrapper) InitCluster(
ctx context.Context, autoscalingNodeGroups []string, cloudServiceAccountURI, k8sVersion string, ctx context.Context, autoscalingNodeGroups []string, cloudServiceAccountURI, versionString string,
id attestationtypes.ID, kmsConfig KMSConfig, sshUsers map[string]string, log *logger.Logger, id attestationtypes.ID, kmsConfig KMSConfig, sshUsers map[string]string, log *logger.Logger,
) ([]byte, error) { ) ([]byte, error) {
log.With(zap.String("version", k8sVersion)).Infof("Installing Kubernetes components") k8sVersion, err := versions.NewValidK8sVersion(versionString)
// InstallComponents validates the k8sVersion as it's first action and returns if not supported. if err != nil {
// This implicitly makes k8sVersion safe to use in this function. return nil, err
}
log.With(zap.String("version", string(k8sVersion))).Infof("Installing Kubernetes components")
if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil { if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil {
return nil, err return nil, err
} }
@ -236,8 +238,12 @@ func (k *KubeWrapper) InitCluster(
} }
// JoinCluster joins existing Kubernetes cluster. // JoinCluster joins existing Kubernetes cluster.
func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, k8sVersion string, log *logger.Logger) error { func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, versionString string, log *logger.Logger) error {
log.With(zap.String("version", k8sVersion)).Infof("Installing Kubernetes components") k8sVersion, err := versions.NewValidK8sVersion(versionString)
if err != nil {
return err
}
log.With(zap.String("version", string(k8sVersion))).Infof("Installing Kubernetes components")
if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil { if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil {
return err return err
} }
@ -311,7 +317,7 @@ func (k *KubeWrapper) setupJoinService(csp string, measurementsJSON []byte, id a
return k.clusterUtil.SetupJoinService(k.client, joinConfiguration) return k.clusterUtil.SetupJoinService(k.client, joinConfiguration)
} }
func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServiceAccountURI string, instance metadata.InstanceMetadata, k8sVersion string) error { func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServiceAccountURI string, instance metadata.InstanceMetadata, k8sVersion versions.ValidK8sVersion) error {
if !k.cloudControllerManager.Supported() { if !k.cloudControllerManager.Supported() {
return nil return nil
} }
@ -339,7 +345,7 @@ func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServ
return nil return nil
} }
func (k *KubeWrapper) setupCloudNodeManager(k8sVersion string) error { func (k *KubeWrapper) setupCloudNodeManager(k8sVersion versions.ValidK8sVersion) error {
if !k.cloudNodeManager.Supported() { if !k.cloudNodeManager.Supported() {
return nil return nil
} }
@ -358,7 +364,7 @@ func (k *KubeWrapper) setupCloudNodeManager(k8sVersion string) error {
return nil return nil
} }
func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata, cloudServiceAccountURI string, autoscalingNodeGroups []string, k8sVersion string) error { func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata, cloudServiceAccountURI string, autoscalingNodeGroups []string, k8sVersion versions.ValidK8sVersion) error {
if !k.clusterAutoscaler.Supported() { if !k.clusterAutoscaler.Supported() {
return nil return nil
} }
@ -377,11 +383,7 @@ func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata,
} }
// setupK8sVersionConfigMap applies a ConfigMap (cf. server-side apply) to consistently store the installed k8s version. // setupK8sVersionConfigMap applies a ConfigMap (cf. server-side apply) to consistently store the installed k8s version.
func (k *KubeWrapper) setupK8sVersionConfigMap(ctx context.Context, k8sVersion string) error { func (k *KubeWrapper) setupK8sVersionConfigMap(ctx context.Context, k8sVersion versions.ValidK8sVersion) error {
if !versions.IsSupportedK8sVersion(k8sVersion) {
return fmt.Errorf("supplied k8s version is not supported: %v", k8sVersion)
}
config := corev1.ConfigMap{ config := corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{ TypeMeta: metav1.TypeMeta{
APIVersion: "v1", APIVersion: "v1",
@ -392,14 +394,14 @@ func (k *KubeWrapper) setupK8sVersionConfigMap(ctx context.Context, k8sVersion s
Namespace: "kube-system", Namespace: "kube-system",
}, },
Data: map[string]string{ Data: map[string]string{
constants.K8sVersion: k8sVersion, constants.K8sVersion: string(k8sVersion),
}, },
} }
// We do not use the client's Apply method here since we are handling a kubernetes-native type. // We do not use the client's Apply method here since we are handling a kubernetes-native type.
// These types don't implement our custom Marshaler interface. // These types don't implement our custom Marshaler interface.
if err := k.client.CreateConfigMap(ctx, config); err != nil { if err := k.client.CreateConfigMap(ctx, config); err != nil {
return fmt.Errorf("Apply in KubeWrapper.setupK8sVersionConfigMap(..) failed with: %v", err) return fmt.Errorf("apply in KubeWrapper.setupK8sVersionConfigMap(..) failed with: %v", err)
} }
return nil return nil

View File

@ -13,6 +13,7 @@ import (
attestationtypes "github.com/edgelesssys/constellation/internal/attestation/types" attestationtypes "github.com/edgelesssys/constellation/internal/attestation/types"
"github.com/edgelesssys/constellation/internal/cloud/metadata" "github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/logger" "github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.uber.org/goleak" "go.uber.org/goleak"
@ -47,7 +48,7 @@ func TestInitCluster(t *testing.T) {
kubeconfigReader configReader kubeconfigReader configReader
wantConfig k8sapi.KubeadmInitYAML wantConfig k8sapi.KubeadmInitYAML
wantErr bool wantErr bool
k8sVersion string k8sVersion versions.ValidK8sVersion
}{ }{
"kubeadm init works without metadata": { "kubeadm init works without metadata": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
@ -70,7 +71,7 @@ func TestInitCluster(t *testing.T) {
}, },
ClusterConfiguration: kubeadm.ClusterConfiguration{}, ClusterConfiguration: kubeadm.ClusterConfiguration{},
}, },
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init works with metadata and loadbalancer": { "kubeadm init works with metadata and loadbalancer": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
@ -110,7 +111,7 @@ func TestInitCluster(t *testing.T) {
}, },
}, },
wantErr: false, wantErr: false,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init fails when retrieving metadata self": { "kubeadm init fails when retrieving metadata self": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
@ -125,7 +126,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{}, CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{}, ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true, wantErr: true,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init fails when retrieving metadata subnetwork cidr": { "kubeadm init fails when retrieving metadata subnetwork cidr": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
@ -140,7 +141,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{}, CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{}, ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true, wantErr: true,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init fails when retrieving metadata loadbalancer ip": { "kubeadm init fails when retrieving metadata loadbalancer ip": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
@ -156,7 +157,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{}, CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{}, ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true, wantErr: true,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init fails when applying the init config": { "kubeadm init fails when applying the init config": {
clusterUtil: stubClusterUtil{initClusterErr: someErr}, clusterUtil: stubClusterUtil{initClusterErr: someErr},
@ -168,7 +169,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{}, CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{}, ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true, wantErr: true,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init fails when setting up the pod network": { "kubeadm init fails when setting up the pod network": {
clusterUtil: stubClusterUtil{setupPodNetworkErr: someErr}, clusterUtil: stubClusterUtil{setupPodNetworkErr: someErr},
@ -180,7 +181,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{}, CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{}, ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true, wantErr: true,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init fails when setting up the join service": { "kubeadm init fails when setting up the join service": {
clusterUtil: stubClusterUtil{setupJoinServiceError: someErr}, clusterUtil: stubClusterUtil{setupJoinServiceError: someErr},
@ -192,7 +193,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{}, CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{}, ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true, wantErr: true,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init fails when setting the cloud contoller manager": { "kubeadm init fails when setting the cloud contoller manager": {
clusterUtil: stubClusterUtil{setupCloudControllerManagerError: someErr}, clusterUtil: stubClusterUtil{setupCloudControllerManagerError: someErr},
@ -204,7 +205,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{}, CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{}, ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true, wantErr: true,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init fails when setting the cloud node manager": { "kubeadm init fails when setting the cloud node manager": {
clusterUtil: stubClusterUtil{setupCloudNodeManagerError: someErr}, clusterUtil: stubClusterUtil{setupCloudNodeManagerError: someErr},
@ -216,7 +217,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{SupportedResp: true}, CloudNodeManager: &stubCloudNodeManager{SupportedResp: true},
ClusterAutoscaler: &stubClusterAutoscaler{}, ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true, wantErr: true,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init fails when setting the cluster autoscaler": { "kubeadm init fails when setting the cluster autoscaler": {
clusterUtil: stubClusterUtil{setupAutoscalingError: someErr}, clusterUtil: stubClusterUtil{setupAutoscalingError: someErr},
@ -228,7 +229,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{}, CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{SupportedResp: true}, ClusterAutoscaler: &stubClusterAutoscaler{SupportedResp: true},
wantErr: true, wantErr: true,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init fails when reading kubeconfig": { "kubeadm init fails when reading kubeconfig": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
@ -240,7 +241,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{}, CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{}, ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true, wantErr: true,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init fails when setting up the kms": { "kubeadm init fails when setting up the kms": {
clusterUtil: stubClusterUtil{setupKMSError: someErr}, clusterUtil: stubClusterUtil{setupKMSError: someErr},
@ -252,7 +253,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{SupportedResp: false}, CloudNodeManager: &stubCloudNodeManager{SupportedResp: false},
ClusterAutoscaler: &stubClusterAutoscaler{}, ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true, wantErr: true,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"kubeadm init fails when setting up verification service": { "kubeadm init fails when setting up verification service": {
clusterUtil: stubClusterUtil{setupVerificationServiceErr: someErr}, clusterUtil: stubClusterUtil{setupVerificationServiceErr: someErr},
@ -264,7 +265,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{SupportedResp: false}, CloudNodeManager: &stubCloudNodeManager{SupportedResp: false},
ClusterAutoscaler: &stubClusterAutoscaler{}, ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true, wantErr: true,
k8sVersion: "1.23", k8sVersion: versions.Latest,
}, },
"unsupported k8sVersion fails cluster creation": { "unsupported k8sVersion fails cluster creation": {
clusterUtil: stubClusterUtil{}, clusterUtil: stubClusterUtil{},
@ -275,7 +276,7 @@ func TestInitCluster(t *testing.T) {
CloudControllerManager: &stubCloudControllerManager{}, CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{}, CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{}, ClusterAutoscaler: &stubClusterAutoscaler{},
k8sVersion: "invalid version", k8sVersion: "1.19",
wantErr: true, wantErr: true,
}, },
} }
@ -296,7 +297,7 @@ func TestInitCluster(t *testing.T) {
kubeconfigReader: tc.kubeconfigReader, kubeconfigReader: tc.kubeconfigReader,
getIPAddr: func() (string, error) { return privateIP, nil }, getIPAddr: func() (string, error) { return privateIP, nil },
} }
_, err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountURI, tc.k8sVersion, attestationtypes.ID{}, KMSConfig{MasterSecret: masterSecret}, nil, logger.NewTest(t)) _, err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountURI, string(tc.k8sVersion), attestationtypes.ID{}, KMSConfig{MasterSecret: masterSecret}, nil, logger.NewTest(t))
if tc.wantErr { if tc.wantErr {
assert.Error(err) assert.Error(err)
@ -321,8 +322,7 @@ func TestJoinCluster(t *testing.T) {
} }
privateIP := "192.0.2.1" privateIP := "192.0.2.1"
// stubClusterUtil does not validate the k8sVersion, thus it can be arbitrary. k8sVersion := versions.Latest
k8sVersion := "3.2.1"
testCases := map[string]struct { testCases := map[string]struct {
clusterUtil stubClusterUtil clusterUtil stubClusterUtil
@ -454,7 +454,7 @@ func TestJoinCluster(t *testing.T) {
getIPAddr: func() (string, error) { return privateIP, nil }, getIPAddr: func() (string, error) { return privateIP, nil },
} }
err := kube.JoinCluster(context.Background(), joinCommand, tc.role, k8sVersion, logger.NewTest(t)) err := kube.JoinCluster(context.Background(), joinCommand, tc.role, string(k8sVersion), logger.NewTest(t))
if tc.wantErr { if tc.wantErr {
assert.Error(err) assert.Error(err)
return return
@ -522,7 +522,7 @@ type stubClusterUtil struct {
joinConfigs [][]byte joinConfigs [][]byte
} }
func (s *stubClusterUtil) InstallComponents(ctx context.Context, version string) error { func (s *stubClusterUtil) InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error {
return s.installComponentsErr return s.installComponentsErr
} }
@ -588,7 +588,7 @@ type stubConfigProvider struct {
JoinConfig k8sapi.KubeadmJoinYAML JoinConfig k8sapi.KubeadmJoinYAML
} }
func (s *stubConfigProvider) InitConfiguration(_ bool, _ string) k8sapi.KubeadmInitYAML { func (s *stubConfigProvider) InitConfiguration(_ bool, _ versions.ValidK8sVersion) k8sapi.KubeadmInitYAML {
return s.InitConfig return s.InitConfig
} }

View File

@ -230,7 +230,7 @@ func Default() *Config {
Measurements: qemuPCRs, Measurements: qemuPCRs,
}, },
}, },
KubernetesVersion: "1.23", KubernetesVersion: string(versions.Latest),
} }
} }

View File

@ -97,8 +97,6 @@ const (
// Kubernetes. // Kubernetes.
// //
// KubernetesVersion installed by kubeadm.
KubernetesVersion = "stable-1.23"
KubernetesJoinTokenTTL = 15 * time.Minute KubernetesJoinTokenTTL = 15 * time.Minute
) )

View File

@ -1,5 +1,28 @@
package versions package versions
import "fmt"
// ValidK8sVersion represents any of the three currently supported k8s versions.
type ValidK8sVersion string
// NewValidK8sVersion validates the given string and produces a new ValidK8sVersion object.
func NewValidK8sVersion(k8sVersion string) (ValidK8sVersion, error) {
if IsSupportedK8sVersion(k8sVersion) {
return ValidK8sVersion(k8sVersion), nil
}
return "", fmt.Errorf("invalid k8sVersion supplied: %s", k8sVersion)
}
// IsSupportedK8sVersion checks if a given Kubernetes version is supported by Constellation.
func IsSupportedK8sVersion(version string) bool {
switch version {
case string(V1_23):
return true
default:
return false
}
}
const ( const (
// Constellation images. // Constellation images.
// These images are built in a way that they support all versions currently listed in VersionConfigs. // These images are built in a way that they support all versions currently listed in VersionConfigs.
@ -8,11 +31,17 @@ const (
KmsImage = "ghcr.io/edgelesssys/constellation/kmsserver:v1.3.2-0.20220714151638-d295be31" KmsImage = "ghcr.io/edgelesssys/constellation/kmsserver:v1.3.2-0.20220714151638-d295be31"
VerificationImage = "ghcr.io/edgelesssys/constellation/verification-service:v1.3.2-0.20220714151638-d295be31" VerificationImage = "ghcr.io/edgelesssys/constellation/verification-service:v1.3.2-0.20220714151638-d295be31"
GcpGuestImage = "ghcr.io/edgelesssys/gcp-guest-agent:latest" GcpGuestImage = "ghcr.io/edgelesssys/gcp-guest-agent:latest"
// currently supported versions.
V1_23 ValidK8sVersion = "1.23"
Latest ValidK8sVersion = V1_23
) )
// versionConfigs holds download URLs for all required kubernetes components for every supported version. // versionConfigs holds download URLs for all required kubernetes components for every supported version.
var VersionConfigs map[string]KubernetesVersion = map[string]KubernetesVersion{ var VersionConfigs map[ValidK8sVersion]KubernetesVersion = map[ValidK8sVersion]KubernetesVersion{
"1.23": { // TODO:
V1_23: {
PatchVersion: "1.23.6",
CNIPluginsURL: "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz", CNIPluginsURL: "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz",
CrictlURL: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.1/crictl-v1.24.1-linux-amd64.tar.gz", CrictlURL: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.1/crictl-v1.24.1-linux-amd64.tar.gz",
KubeletServiceURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service", KubeletServiceURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service",
@ -20,8 +49,6 @@ var VersionConfigs map[string]KubernetesVersion = map[string]KubernetesVersion{
KubeletURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubelet", KubeletURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubelet",
KubeadmURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubeadm", KubeadmURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubeadm",
KubectlURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl", KubectlURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl",
// CloudControllerManagerImageAWS is the CCM image used on AWS.
CloudControllerManagerImageAWS: "us.gcr.io/k8s-artifacts-prod/provider-aws/cloud-controller-manager:v1.22.0-alpha.0",
// CloudControllerManagerImageGCP is the CCM image used on GCP. // CloudControllerManagerImageGCP is the CCM image used on GCP.
// TODO: use newer "cloud-provider-gcp" from https://github.com/kubernetes/cloud-provider-gcp when newer releases are available. // TODO: use newer "cloud-provider-gcp" from https://github.com/kubernetes/cloud-provider-gcp when newer releases are available.
CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:sha-2f6a5b07fc2d37f24f8ff725132f87584d627d8f", CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:sha-2f6a5b07fc2d37f24f8ff725132f87584d627d8f",
@ -36,6 +63,7 @@ var VersionConfigs map[string]KubernetesVersion = map[string]KubernetesVersion{
// KubernetesVersion bundles download URLs to all version-releated binaries necessary for installing/deploying a particular Kubernetes version. // KubernetesVersion bundles download URLs to all version-releated binaries necessary for installing/deploying a particular Kubernetes version.
type KubernetesVersion struct { type KubernetesVersion struct {
PatchVersion string
CNIPluginsURL string CNIPluginsURL string
CrictlURL string CrictlURL string
KubeletServiceURL string KubeletServiceURL string
@ -49,9 +77,3 @@ type KubernetesVersion struct {
CloudNodeManagerImageAzure string CloudNodeManagerImageAzure string
ClusterAutoscalerImage string ClusterAutoscalerImage string
} }
// IsSupportedK8sVersion checks if a given Kubernetes version is supported by Constellation.
func IsSupportedK8sVersion(version string) bool {
_, ok := VersionConfigs[version]
return ok
}

View File

@ -12,6 +12,7 @@ import (
"github.com/edgelesssys/constellation/internal/constants" "github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/file" "github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/logger" "github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/edgelesssys/constellation/joinservice/joinproto" "github.com/edgelesssys/constellation/joinservice/joinproto"
"github.com/spf13/afero" "github.com/spf13/afero"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -37,7 +38,7 @@ func TestIssueJoinTicket(t *testing.T) {
CACertHashes: []string{"hash"}, CACertHashes: []string{"hash"},
Token: "token", Token: "token",
} }
testK8sVersion := "1.23" testK8sVersion := versions.Latest
testCases := map[string]struct { testCases := map[string]struct {
isControlPlane bool isControlPlane bool