Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-12-11 17:04:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

AB#2074: Choosable K8S Version (#277)

Browse Source 
AB#2074: Add configurable k8s version

Configurable version flow:
* cli config holds/validates k8sVersion
* InitCluster receive a k8sVersion arg
* InitCluster creates CM "k8s-version"
* kubeadm's InitConfiguration receives k8sVersion
* joinservice spec mounts/reads k8s-version CM
* joinservice supplies k8sVersion via JoinTicketResponse
Other changes:
* Remove unused test code (FakeK8SClient)
* move VersionConfig map to /internal/versions
* installk8sComponents is now a function instead of a method
This commit is contained in:
Otto Bittner 2022-07-18 12:28:02 +02:00 committed by GitHub
parent d3466da393
commit a68ee817ff
31 changed files with 360 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bootstrapper/cmd/bootstrapper/test.go
View File

@ -21,7 +21,7 @@ func (c *clusterFake) InitCluster(context.Context, []string, string, string, att
}
// JoinCluster will fake joining the current node to an existing cluster.
func (c *clusterFake) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, *logger.Logger) error {
func (c *clusterFake) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, string, *logger.Logger) error {
return nil
}

3
bootstrapper/internal/joinclient/client.go
View File

@ -272,7 +272,7 @@ func (c *JoinClient) startNodeAndJoin(ticket *joinproto.IssueJoinTicketResponse,
Token: ticket.Token,
CACertHashes: []string{ticket.DiscoveryTokenCaCertHash},
}
if err := c.joiner.JoinCluster(ctx, btd, c.role, c.log); err != nil {
if err := c.joiner.JoinCluster(ctx, btd, c.role, ticket.KubernetesVersion, c.log); err != nil {
return fmt.Errorf("joining Kubernetes cluster: %w", err)
}
@ -386,6 +386,7 @@ type ClusterJoiner interface {
ctx context.Context,
args *kubeadm.BootstrapTokenDiscovery,
peerRole role.Role,
k8sVersion string,
log *logger.Logger,
) error
}

2
bootstrapper/internal/joinclient/client_test.go
View File

@ -385,7 +385,7 @@ type stubClusterJoiner struct {
joinClusterErr error
}
func (j *stubClusterJoiner) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, *logger.Logger) error {
func (j *stubClusterJoiner) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, string, *logger.Logger) error {
j.joinClusterCalled = true
return j.joinClusterErr
}

16
bootstrapper/internal/kubernetes/k8sapi/constants.go Normal file
View File

@ -0,0 +1,16 @@
package k8sapi
const (
// Paths and permissions necessary for Kubernetes installation.
cniPluginsDir = "/opt/cni/bin"
binDir = "/run/state/bin"
kubeadmPath = "/run/state/bin/kubeadm"
kubeletPath = "/run/state/bin/kubelet"
kubectlPath = "/run/state/bin/kubectl"
kubeletServiceEtcPath = "/etc/systemd/system/kubelet.service"
kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service"
kubeadmConfEtcPath = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
kubeadmConfStatePath = "/run/state/systemd/system/kubelet.service.d/10-kubeadm.conf"
executablePerm = 0o544
systemdUnitPerm = 0o644
)

7
bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go
View File

@ -5,7 +5,6 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/internal/kubelet"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/internal/constants"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletconf "k8s.io/kubelet/config/v1beta1"
@ -25,7 +24,7 @@ const (
type CoreOSConfiguration struct{}
func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) KubeadmInitYAML {
func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool, k8sVersion string) KubeadmInitYAML {
var cloudProvider string
if externalCloudProvider {
cloudProvider = "external"
@ -48,12 +47,14 @@ func (c *CoreOSConfiguration) InitConfiguration(externalCloudProvider bool) Kube
BindPort: bindPort,
},
},
// https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3#ClusterConfiguration
ClusterConfiguration: kubeadm.ClusterConfiguration{
TypeMeta: metav1.TypeMeta{
Kind: "ClusterConfiguration",
APIVersion: kubeadm.SchemeGroupVersion.String(),
},
KubernetesVersion: constants.KubernetesVersion,
// Target kubernetes version of the control plane.
KubernetesVersion: k8sVersion,
// necessary to be able to access the kubeapi server through localhost
APIServer: kubeadm.APIServer{
ControlPlaneComponent: kubeadm.ControlPlaneComponent{

4
bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go
View File

@ -19,11 +19,11 @@ func TestInitConfiguration(t *testing.T) {
config KubeadmInitYAML
}{
"CoreOS init config can be created": {
config: coreOSConfig.InitConfiguration(true),
config: coreOSConfig.InitConfiguration(true, "3.2.1"),
},
"CoreOS init config with all fields can be created": {
config: func() KubeadmInitYAML {
c := coreOSConfig.InitConfiguration(true)
c := coreOSConfig.InitConfiguration(true, "3.2.1")
c.SetAPIServerAdvertiseAddress("192.0.2.0")
c.SetNodeIP("192.0.2.0")
c.SetNodeName("node")

12
bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go
View File

@ -2,9 +2,11 @@ package client
import (
"bytes"
"context"
"fmt"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
@ -88,3 +90,13 @@ func (c *Client) GetObjects(resources resources.Marshaler) ([]*resource.Info, er
Do()
return result.Infos()
}
// CreateConfigMap creates the given ConfigMap.
func (c *Client) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
_, err := c.clientset.CoreV1().ConfigMaps(configMap.ObjectMeta.Namespace).Create(ctx, &configMap, metav1.CreateOptions{})
if err != nil {
return err
}
return nil
}

17
bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go
View File

@ -1,10 +1,12 @@
package kubectl
import (
"context"
"errors"
"fmt"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
corev1 "k8s.io/api/core/v1"
"k8s.io/cli-runtime/pkg/resource"
)
@ -17,6 +19,7 @@ type Client interface {
ApplyOneObject(info *resource.Info, forceConflicts bool) error
// GetObjects converts resources into prepared info fields for use in ApplyOneObject.
GetObjects(resources resources.Marshaler) ([]*resource.Info, error)
CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
}
// clientGenerator can generate new clients from a kubeconfig.
@ -66,3 +69,17 @@ func (k *Kubectl) Apply(resources resources.Marshaler, forceConflicts bool) erro
func (k *Kubectl) SetKubeconfig(kubeconfig []byte) {
k.kubeconfig = kubeconfig
}
func (k *Kubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
client, err := k.clientGenerator.NewClient(k.kubeconfig)
if err != nil {
return err
}
err = client.CreateConfigMap(ctx, configMap)
if err != nil {
return err
}
return nil
}

23
bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go
View File

@ -1,12 +1,14 @@
package kubectl
import (
"context"
"errors"
"testing"
"github.com/edgelesssys/constellation/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/stretchr/testify/assert"
"go.uber.org/goleak"
corev1 "k8s.io/api/core/v1"
"k8s.io/cli-runtime/pkg/resource"
)
@ -15,9 +17,10 @@ func TestMain(m *testing.M) {
}
type stubClient struct {
applyOneObjectErr error
getObjectsInfos []*resource.Info
getObjectsErr error
applyOneObjectErr error
getObjectsInfos []*resource.Info
getObjectsErr error
createConfigMapErr error
}
func (s *stubClient) ApplyOneObject(info *resource.Info, forceConflicts bool) error {
@ -28,11 +31,16 @@ func (s *stubClient) GetObjects(resources resources.Marshaler) ([]*resource.Info
return s.getObjectsInfos, s.getObjectsErr
}
func (s *stubClient) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
return s.createConfigMapErr
}
type stubClientGenerator struct {
applyOneObjectErr error
getObjectsInfos []*resource.Info
getObjectsErr error
newClientErr error
applyOneObjectErr error
getObjectsInfos []*resource.Info
getObjectsErr error
newClientErr error
createConfigMapErr error
}
func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) {
@ -40,6 +48,7 @@ func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) {
s.applyOneObjectErr,
s.getObjectsInfos,
s.getObjectsErr,
s.createConfigMapErr,
}, s.newClientErr
}

3
bootstrapper/internal/kubernetes/k8sapi/resources/access_manager.go
View File

@ -2,6 +2,7 @@ package resources
import (
"github.com/edgelesssys/constellation/internal/secrets"
"github.com/edgelesssys/constellation/internal/versions"
"google.golang.org/protobuf/proto"
apps "k8s.io/api/apps/v1"
k8s "k8s.io/api/core/v1"
@ -104,7 +105,7 @@ func NewAccessManagerDeployment(sshUsers map[string]string) *accessManagerDeploy
InitContainers: []k8s.Container{
{
Name: "constellation-access-manager",
Image: accessManagerImage,
Image: versions.AccessManagerImage,
VolumeMounts: []k8s.VolumeMount{
{
Name: "host",

3
bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go
View File

@ -1,6 +1,7 @@
package resources
import (
"github.com/edgelesssys/constellation/internal/versions"
"google.golang.org/protobuf/proto"
apps "k8s.io/api/apps/v1"
k8s "k8s.io/api/core/v1"
@ -434,7 +435,7 @@ func NewDefaultAutoscalerDeployment(extraVolumes []k8s.Volume, extraVolumeMounts
Containers: []k8s.Container{
{
Name: "cluster-autoscaler",
Image: clusterAutoscalerImage,
Image: versions.ClusterAutoscalerImage,
ImagePullPolicy: k8s.PullIfNotPresent,
LivenessProbe: &k8s.Probe{
ProbeHandler: k8s.ProbeHandler{

3
bootstrapper/internal/kubernetes/k8sapi/resources/gcp_guest_agent.go
View File

@ -2,6 +2,7 @@ package resources
import (
"github.com/edgelesssys/constellation/internal/secrets"
"github.com/edgelesssys/constellation/internal/versions"
apps "k8s.io/api/apps/v1"
k8s "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -64,7 +65,7 @@ func NewGCPGuestAgentDaemonset() *gcpGuestAgentDaemonset {
Containers: []k8s.Container{
{
Name: "gcp-guest-agent",
Image: gcpGuestImage, // built from https://github.com/edgelesssys/gcp-guest-agent
Image: versions.GcpGuestImage, // built from https://github.com/edgelesssys/gcp-guest-agent
SecurityContext: &k8s.SecurityContext{
Privileged: func(b bool) *bool { return &b }(true),
Capabilities: &k8s.Capabilities{

13
bootstrapper/internal/kubernetes/k8sapi/resources/images.go
View File

@ -1,13 +0,0 @@
package resources
const (
// Constellation images.
joinImage = "ghcr.io/edgelesssys/constellation/join-service:v1.3.2-0.20220714151638-d295be31"
accessManagerImage = "ghcr.io/edgelesssys/constellation/access-manager:v1.3.2-0.20220714151638-d295be31"
kmsImage = "ghcr.io/edgelesssys/constellation/kmsserver:v1.3.2-0.20220714151638-d295be31"
verificationImage = "ghcr.io/edgelesssys/constellation/verification-service:v1.3.2-0.20220714151638-d295be31"
gcpGuestImage = "ghcr.io/edgelesssys/gcp-guest-agent:latest"
// external images.
clusterAutoscalerImage = "k8s.gcr.io/autoscaling/cluster-autoscaler:v1.23.0"
)

22
bootstrapper/internal/kubernetes/k8sapi/resources/joinservice.go
View File

@ -5,6 +5,7 @@ import (
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/secrets"
"github.com/edgelesssys/constellation/internal/versions"
apps "k8s.io/api/apps/v1"
k8s "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
@ -135,7 +136,7 @@ func NewJoinServiceDaemonset(csp string, measurementsJSON, idJSON string) *joinS
Containers: []k8s.Container{
{
Name: "join-service",
Image: joinImage,
Image: versions.JoinImage,
Ports: []k8s.ContainerPort{
{
ContainerPort: constants.JoinServicePort,
@ -167,9 +168,22 @@ func NewJoinServiceDaemonset(csp string, measurementsJSON, idJSON string) *joinS
{
Name: "config",
VolumeSource: k8s.VolumeSource{
ConfigMap: &k8s.ConfigMapVolumeSource{
LocalObjectReference: k8s.LocalObjectReference{
Name: "join-config",
Projected: &k8s.ProjectedVolumeSource{
Sources: []k8s.VolumeProjection{
{
ConfigMap: &k8s.ConfigMapProjection{
LocalObjectReference: k8s.LocalObjectReference{
Name: "join-config",
},
},
},
{
ConfigMap: &k8s.ConfigMapProjection{
LocalObjectReference: k8s.LocalObjectReference{
Name: constants.K8sVersion,
},
},
},
},
},
},

3
bootstrapper/internal/kubernetes/k8sapi/resources/kms.go
View File

@ -5,6 +5,7 @@ import (
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/secrets"
"github.com/edgelesssys/constellation/internal/versions"
apps "k8s.io/api/apps/v1"
k8s "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
@ -226,7 +227,7 @@ func NewKMSDeployment(csp string, masterSecret []byte) *kmsDeployment {
Containers: []k8s.Container{
{
Name: "kms",
Image: kmsImage,
Image: versions.KmsImage,
Args: []string{
fmt.Sprintf("--atls-port=%d", constants.KMSATLSPort),
fmt.Sprintf("--port=%d", constants.KMSPort),

3
bootstrapper/internal/kubernetes/k8sapi/resources/verification.go
View File

@ -5,6 +5,7 @@ import (
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/secrets"
"github.com/edgelesssys/constellation/internal/versions"
apps "k8s.io/api/apps/v1"
k8s "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -73,7 +74,7 @@ func NewVerificationDaemonSet(csp string) *verificationDaemonset {
Containers: []k8s.Container{
{
Name: "verification-service",
Image: verificationImage,
Image: versions.VerificationImage,
Ports: []k8s.ContainerPort{
{
Name: "http",

54
bootstrapper/internal/kubernetes/k8sapi/util.go
View File

@ -7,6 +7,7 @@ import (
"encoding/pem"
"errors"
"fmt"
"io/fs"
"net"
"net/http"
"os"
@ -21,8 +22,12 @@ import (
"github.com/edgelesssys/constellation/bootstrapper/util"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/icholy/replace"
"github.com/spf13/afero"
"go.uber.org/zap"
"golang.org/x/text/transform"
corev1 "k8s.io/api/core/v1"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
@ -39,9 +44,17 @@ var providerIDRegex = regexp.MustCompile(`^azure:///subscriptions/([^/]+)/resour
type Client interface {
Apply(resources resources.Marshaler, forceConflicts bool) error
SetKubeconfig(kubeconfig []byte)
CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
// TODO: add tolerations
}
type installer interface {
Install(
ctx context.Context, sourceURL string, destinations []string, perm fs.FileMode,
extract bool, transforms ...transform.Transformer,
) error
}
// KubernetesUtil provides low level management of the kubernetes cluster.
type KubernetesUtil struct {
inst installer
@ -58,13 +71,46 @@ func NewKubernetesUtil() *KubernetesUtil {
// InstallComponents installs kubernetes components in the version specified.
func (k *KubernetesUtil) InstallComponents(ctx context.Context, version string) error {
var versionConf kubernetesVersion
var versionConf versions.KubernetesVersion
var ok bool
if versionConf, ok = versionConfigs[version]; !ok {
if versionConf, ok = versions.VersionConfigs[version]; !ok {
return fmt.Errorf("unsupported kubernetes version %q", version)
}
if err := versionConf.installK8sComponents(ctx, k.inst); err != nil {
return err
if err := k.inst.Install(
ctx, versionConf.CNIPluginsURL, []string{cniPluginsDir}, executablePerm, true,
); err != nil {
return fmt.Errorf("installing cni plugins: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.CrictlURL, []string{binDir}, executablePerm, true,
); err != nil {
return fmt.Errorf("installing crictl: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.KubeletServiceURL, []string{kubeletServiceEtcPath, kubeletServiceStatePath}, systemdUnitPerm, false, replace.String("/usr/bin", binDir),
); err != nil {
return fmt.Errorf("installing kubelet service: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.KubeadmConfURL, []string{kubeadmConfEtcPath, kubeadmConfStatePath}, systemdUnitPerm, false, replace.String("/usr/bin", binDir),
); err != nil {
return fmt.Errorf("installing kubeadm conf: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.KubeletURL, []string{kubeletPath}, executablePerm, false,
); err != nil {
return fmt.Errorf("installing kubelet: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.KubeadmURL, []string{kubeadmPath}, executablePerm, false,
); err != nil {
return fmt.Errorf("installing kubeadm: %w", err)
}
if err := k.inst.Install(
ctx, versionConf.KubectlURL, []string{kubectlPath}, executablePerm, false,
); err != nil {
return fmt.Errorf("installing kubectl: %w", err)
}
return enableSystemdUnit(ctx, kubeletServiceEtcPath)

95
bootstrapper/internal/kubernetes/k8sapi/versions.go
View File

@ -1,95 +0,0 @@
package k8sapi
import (
"context"
"fmt"
"io/fs"
"github.com/icholy/replace"
"golang.org/x/text/transform"
)
const (
cniPluginsDir = "/opt/cni/bin"
binDir = "/run/state/bin"
kubeadmPath = "/run/state/bin/kubeadm"
kubeletPath = "/run/state/bin/kubelet"
kubectlPath = "/run/state/bin/kubectl"
kubeletServiceEtcPath = "/etc/systemd/system/kubelet.service"
kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service"
kubeadmConfEtcPath = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
kubeadmConfStatePath = "/run/state/systemd/system/kubelet.service.d/10-kubeadm.conf"
executablePerm = 0o544
systemdUnitPerm = 0o644
)
// versionConfigs holds download URLs for all required kubernetes components for every supported version.
var versionConfigs map[string]kubernetesVersion = map[string]kubernetesVersion{
"1.23.6": {
CNIPluginsURL: "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz",
CrictlURL: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.1/crictl-v1.24.1-linux-amd64.tar.gz",
KubeletServiceURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service",
KubeadmConfURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf",
KubeletURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubelet",
KubeadmURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubeadm",
KubectlURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl",
},
}
type kubernetesVersion struct {
CNIPluginsURL string
CrictlURL string
KubeletServiceURL string
KubeadmConfURL string
KubeletURL string
KubeadmURL string
KubectlURL string
}
// installK8sComponents installs kubernetes components for this version.
// reference: https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#installing-kubeadm-kubelet-and-kubectl .
func (k *kubernetesVersion) installK8sComponents(ctx context.Context, inst installer) error {
if err := inst.Install(
ctx, k.CNIPluginsURL, []string{cniPluginsDir}, executablePerm, true,
); err != nil {
return fmt.Errorf("installing cni plugins: %w", err)
}
if err := inst.Install(
ctx, k.CrictlURL, []string{binDir}, executablePerm, true,
); err != nil {
return fmt.Errorf("installing crictl: %w", err)
}
if err := inst.Install(
ctx, k.KubeletServiceURL, []string{kubeletServiceEtcPath, kubeletServiceStatePath}, systemdUnitPerm, false, replace.String("/usr/bin", binDir),
); err != nil {
return fmt.Errorf("installing kubelet service: %w", err)
}
if err := inst.Install(
ctx, k.KubeadmConfURL, []string{kubeadmConfEtcPath, kubeadmConfStatePath}, systemdUnitPerm, false, replace.String("/usr/bin", binDir),
); err != nil {
return fmt.Errorf("installing kubeadm conf: %w", err)
}
if err := inst.Install(
ctx, k.KubeletURL, []string{kubeletPath}, executablePerm, false,
); err != nil {
return fmt.Errorf("installing kubelet: %w", err)
}
if err := inst.Install(
ctx, k.KubeadmURL, []string{kubeadmPath}, executablePerm, false,
); err != nil {
return fmt.Errorf("installing kubeadm: %w", err)
}
if err := inst.Install(
ctx, k.KubectlURL, []string{kubectlPath}, executablePerm, false,
); err != nil {
return fmt.Errorf("installing kubectl: %w", err)
}
return nil
}
type installer interface {
Install(
ctx context.Context, sourceURL string, destinations []string, perm fs.FileMode,
extract bool, transforms ...transform.Transformer,
) error
}

69
bootstrapper/internal/kubernetes/kubernetes.go
View File

@ -16,8 +16,11 @@ import (
attestationtypes "github.com/edgelesssys/constellation/internal/attestation/types"
"github.com/edgelesssys/constellation/internal/cloud/metadata"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/spf13/afero"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
@ -28,7 +31,7 @@ type configReader interface {
// configurationProvider provides kubeadm init and join configuration.
type configurationProvider interface {
InitConfiguration(externalCloudProvider bool) k8sapi.KubeadmInitYAML
InitConfiguration(externalCloudProvider bool, k8sVersion string) k8sapi.KubeadmInitYAML
JoinConfiguration(externalCloudProvider bool) k8sapi.KubeadmJoinYAML
}
@ -79,7 +82,6 @@ func (k *KubeWrapper) InitCluster(
ctx context.Context, autoscalingNodeGroups []string, cloudServiceAccountURI, k8sVersion string,
id attestationtypes.ID, kmsConfig KMSConfig, sshUsers map[string]string, log *logger.Logger,
) ([]byte, error) {
// TODO: k8s version should be user input
log.With(zap.String("version", k8sVersion)).Infof("Installing Kubernetes components")
if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil {
return nil, err
@ -149,7 +151,7 @@ func (k *KubeWrapper) InitCluster(
).Infof("Setting information for node")
// Step 2: configure kubeadm init config
initConfig := k.configProvider.InitConfiguration(k.cloudControllerManager.Supported())
initConfig := k.configProvider.InitConfiguration(k.cloudControllerManager.Supported(), k8sVersion)
initConfig.SetNodeIP(nodeIP)
initConfig.SetCertSANs([]string{publicIP, nodeIP})
initConfig.SetNodeName(nodeName)
@ -219,16 +221,21 @@ func (k *KubeWrapper) InitCluster(
}
}
// Store the received k8sVersion in a ConfigMap, overwriting exisiting values (there shouldn't be any).
// Joining nodes determine the kubernetes version they will install based on this ConfigMap.
if err := k.setupK8sVersionConfigMap(ctx, k8sVersion, true); err != nil {
return nil, fmt.Errorf("failed to setup k8s version ConfigMap: %v", err)
}
k.clusterUtil.FixCilium(nodeName)
return k.GetKubeconfig()
}
// JoinCluster joins existing Kubernetes cluster.
func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, log *logger.Logger) error {
// TODO: k8s version should be user input
log.With(zap.String("version", "1.23.6")).Infof("Installing Kubernetes components")
if err := k.clusterUtil.InstallComponents(ctx, "1.23.6"); err != nil {
func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, k8sVersion string, log *logger.Logger) error {
log.With(zap.String("version", k8sVersion)).Infof("Installing Kubernetes components")
if err := k.clusterUtil.InstallComponents(ctx, k8sVersion); err != nil {
return err
}
@ -357,6 +364,35 @@ func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata,
return nil
}
// setupK8sVersionConfigMap applies a ConfigMap (cf. server-side apply) to consistently store the installed k8s version.
func (k *KubeWrapper) setupK8sVersionConfigMap(ctx context.Context, k8sVersion string, forceConflicts bool) error {
if !versions.IsSupportedK8sVersion(k8sVersion) {
return fmt.Errorf("supplied k8s version is not supported: %v", k8sVersion)
}
config := corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "k8s-version",
Namespace: "kube-system",
},
Data: map[string]string{
"K8sVersion": k8sVersion,
},
}
// We do not use the client's Apply method here since we are handling a kubernetes-native type.
// These types don't implement our custom Marshaler interface.
if err := k.client.CreateConfigMap(ctx, config); err != nil {
return fmt.Errorf("Apply in KubeWrapper.setupK8sVersionConfigMap(..) failed with: %v", err)
}
return nil
}
// manuallySetLoadbalancerIP sets the loadbalancer IP of the first control plane during init.
// The GCP guest agent does this usually, but is deployed in the cluster that doesn't exist
// at this point. This is a workaround to set the loadbalancer IP manually, so kubeadm and kubelet
@ -391,22 +427,3 @@ func k8sCompliantHostname(in string) string {
func (k *KubeWrapper) StartKubelet() error {
return k.clusterUtil.StartKubelet()
}
type fakeK8SClient struct {
kubeconfig []byte
}
// NewFakeK8SClient creates a new, fake k8s client where apply always works.
func NewFakeK8SClient([]byte) (k8sapi.Client, error) {
return &fakeK8SClient{}, nil
}
// Apply fakes applying Kubernetes resources.
func (f *fakeK8SClient) Apply(resources resources.Marshaler, forceConflicts bool) error {
return nil
}
// SetKubeconfig stores the kubeconfig given to it.
func (f *fakeK8SClient) SetKubeconfig(kubeconfig []byte) {
f.kubeconfig = kubeconfig
}

56
bootstrapper/internal/kubernetes/kubernetes_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
corev1 "k8s.io/api/core/v1"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
@ -35,11 +36,10 @@ func TestInitCluster(t *testing.T) {
publicIP := "192.0.2.2"
loadbalancerIP := "192.0.2.3"
aliasIPRange := "192.0.2.0/24"
k8sVersion := "1.23.8"
testCases := map[string]struct {
clusterUtil stubClusterUtil
kubeCTL stubKubeCTL
kubectl stubKubectl
providerMetadata ProviderMetadata
CloudControllerManager CloudControllerManager
CloudNodeManager CloudNodeManager
@ -47,6 +47,7 @@ func TestInitCluster(t *testing.T) {
kubeconfigReader configReader
wantConfig k8sapi.KubeadmInitYAML
wantErr bool
k8sVersion string
}{
"kubeadm init works without metadata": {
clusterUtil: stubClusterUtil{},
@ -69,6 +70,7 @@ func TestInitCluster(t *testing.T) {
},
ClusterConfiguration: kubeadm.ClusterConfiguration{},
},
k8sVersion: "1.23.6",
},
"kubeadm init works with metadata and loadbalancer": {
clusterUtil: stubClusterUtil{},
@ -107,7 +109,8 @@ func TestInitCluster(t *testing.T) {
},
},
},
wantErr: false,
wantErr: false,
k8sVersion: "1.23.6",
},
"kubeadm init fails when retrieving metadata self": {
clusterUtil: stubClusterUtil{},
@ -122,6 +125,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: "1.23.6",
},
"kubeadm init fails when retrieving metadata subnetwork cidr": {
clusterUtil: stubClusterUtil{},
@ -136,6 +140,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: "1.23.6",
},
"kubeadm init fails when retrieving metadata loadbalancer ip": {
clusterUtil: stubClusterUtil{},
@ -151,6 +156,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: "1.23.6",
},
"kubeadm init fails when applying the init config": {
clusterUtil: stubClusterUtil{initClusterErr: someErr},
@ -162,6 +168,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: "1.23.6",
},
"kubeadm init fails when setting up the pod network": {
clusterUtil: stubClusterUtil{setupPodNetworkErr: someErr},
@ -173,6 +180,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: "1.23.6",
},
"kubeadm init fails when setting up the join service": {
clusterUtil: stubClusterUtil{setupJoinServiceError: someErr},
@ -184,6 +192,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: "1.23.6",
},
"kubeadm init fails when setting the cloud contoller manager": {
clusterUtil: stubClusterUtil{setupCloudControllerManagerError: someErr},
@ -195,6 +204,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: "1.23.6",
},
"kubeadm init fails when setting the cloud node manager": {
clusterUtil: stubClusterUtil{setupCloudNodeManagerError: someErr},
@ -206,6 +216,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{SupportedResp: true},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: "1.23.6",
},
"kubeadm init fails when setting the cluster autoscaler": {
clusterUtil: stubClusterUtil{setupAutoscalingError: someErr},
@ -217,6 +228,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{SupportedResp: true},
wantErr: true,
k8sVersion: "1.23.6",
},
"kubeadm init fails when reading kubeconfig": {
clusterUtil: stubClusterUtil{},
@ -228,6 +240,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: "1.23.6",
},
"kubeadm init fails when setting up the kms": {
clusterUtil: stubClusterUtil{setupKMSError: someErr},
@ -239,6 +252,7 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{SupportedResp: false},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: "1.23.6",
},
"kubeadm init fails when setting up verification service": {
clusterUtil: stubClusterUtil{setupVerificationServiceErr: someErr},
@ -250,6 +264,19 @@ func TestInitCluster(t *testing.T) {
CloudNodeManager: &stubCloudNodeManager{SupportedResp: false},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: "1.23.6",
},
"unsupported k8sVersion fails cluster creation": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
k8sVersion: "invalid version",
wantErr: true,
},
}
@ -265,11 +292,11 @@ func TestInitCluster(t *testing.T) {
cloudNodeManager: tc.CloudNodeManager,
clusterAutoscaler: tc.ClusterAutoscaler,
configProvider: &stubConfigProvider{InitConfig: k8sapi.KubeadmInitYAML{}},
client: &tc.kubeCTL,
client: &tc.kubectl,
kubeconfigReader: tc.kubeconfigReader,
getIPAddr: func() (string, error) { return privateIP, nil },
}
_, err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountURI, k8sVersion, attestationtypes.ID{}, KMSConfig{MasterSecret: masterSecret}, nil, logger.NewTest(t))
_, err := kube.InitCluster(context.Background(), autoscalingNodeGroups, serviceAccountURI, tc.k8sVersion, attestationtypes.ID{}, KMSConfig{MasterSecret: masterSecret}, nil, logger.NewTest(t))
if tc.wantErr {
assert.Error(err)
@ -294,6 +321,8 @@ func TestJoinCluster(t *testing.T) {
}
privateIP := "192.0.2.1"
// stubClusterUtil does not validate the k8sVersion, thus it can be arbitrary.
k8sVersion := "3.2.1"
testCases := map[string]struct {
clusterUtil stubClusterUtil
@ -425,7 +454,7 @@ func TestJoinCluster(t *testing.T) {
getIPAddr: func() (string, error) { return privateIP, nil },
}
err := kube.JoinCluster(context.Background(), joinCommand, tc.role, logger.NewTest(t))
err := kube.JoinCluster(context.Background(), joinCommand, tc.role, k8sVersion, logger.NewTest(t))
if tc.wantErr {
assert.Error(err)
return
@ -559,7 +588,7 @@ type stubConfigProvider struct {
JoinConfig k8sapi.KubeadmJoinYAML
}
func (s *stubConfigProvider) InitConfiguration(_ bool) k8sapi.KubeadmInitYAML {
func (s *stubConfigProvider) InitConfiguration(_ bool, _ string) k8sapi.KubeadmInitYAML {
return s.InitConfig
}
@ -574,22 +603,27 @@ func (s *stubConfigProvider) JoinConfiguration(_ bool) k8sapi.KubeadmJoinYAML {
return s.JoinConfig
}
type stubKubeCTL struct {
ApplyErr error
type stubKubectl struct {
ApplyErr error
createConfigMapErr error
resources []resources.Marshaler
kubeconfigs [][]byte
}
func (s *stubKubeCTL) Apply(resources resources.Marshaler, forceConflicts bool) error {
func (s *stubKubectl) Apply(resources resources.Marshaler, forceConflicts bool) error {
s.resources = append(s.resources, resources)
return s.ApplyErr
}
func (s *stubKubeCTL) SetKubeconfig(kubeconfig []byte) {
func (s *stubKubectl) SetKubeconfig(kubeconfig []byte) {
s.kubeconfigs = append(s.kubeconfigs, kubeconfig)
}
func (s *stubKubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
return s.createConfigMapErr
}
type stubKubeconfigReader struct {
Kubeconfig []byte
ReadErr error

2
cli/internal/cmd/init.go
View File

@ -120,7 +120,7 @@ func initialize(cmd *cobra.Command, dialer grpcDialer, serviceAccCreator service
KeyEncryptionKeyId: "",
UseExistingKek: false,
CloudServiceAccountUri: serviceAccount,
KubernetesVersion: "1.23.6",
KubernetesVersion: config.KubernetesVersion,
SshUserKeys: ssh.ToProtoSlice(sshUsers),
}
resp, err := initCall(cmd.Context(), dialer, stat.BootstrapperHost, req)

1
hack/go.mod
View File

@ -56,6 +56,7 @@ require (
github.com/emirpasic/gods v1.12.0 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
github.com/go-git/go-billy/v5 v5.3.1 // indirect
github.com/icholy/replace v0.5.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect

4
hack/go.sum
View File

@ -636,6 +636,8 @@ github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/icholy/replace v0.5.0 h1:Nx80zYQVlowdba+3Y6dvHDnmxaGtBrDlf2wYn9GyIXQ=
github.com/icholy/replace v0.5.0/go.mod h1:zzi8pxElj2t/5wHHHYmH45D+KxytX/t4w3ClY5nlK+g=
github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
@ -1344,6 +1346,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@ -1654,6 +1657,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 h1:dbuHpmKjkDzSOMKAWl10QNlgaZUd3V1q99xc81tt2Kc=
gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

14
internal/config/config.go
View File

@ -11,6 +11,7 @@ import (
"github.com/edgelesssys/constellation/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/go-playground/locales/en"
ut "github.com/go-playground/universal-translator"
"github.com/go-playground/validator/v10"
@ -60,6 +61,9 @@ type Config struct {
// examples:
// - value: '[]UserKey{ { Username: "Alice", PublicKey: "ssh-rsa AAAAB3NzaC...5QXHKW1rufgtJeSeJ8= alice@domain.com" } }'
SSHUsers []UserKey `yaml:"sshUsers,omitempty" validate:"dive"`
// description: |
// Kubernetes version installed in the cluster.
KubernetesVersion string `yaml:"kubernetesVersion" validate:"supported_k8s_version"`
}
// UserKey describes a user that should be created with corresponding public SSH key.
@ -226,9 +230,14 @@ func Default() *Config {
Measurements: qemuPCRs,
},
},
KubernetesVersion: "1.23.6",
}
}
func validateK8sVersion(fl validator.FieldLevel) bool {
return versions.IsSupportedK8sVersion(fl.Field().String())
}
// Validate checks the config values and returns validation error messages.
// The function only returns an error if the validation itself fails.
func (c *Config) Validate() ([]string, error) {
@ -238,6 +247,11 @@ func (c *Config) Validate() ([]string, error) {
return nil, err
}
// register custom validator with label supported_k8s_version to validate version based on available versionConfigs.
if err := validate.RegisterValidation("supported_k8s_version", validateK8sVersion); err != nil {
return nil, err
}
err := validate.Struct(c)
if err == nil {
return nil, nil

4
internal/constants/constants.go
View File

@ -63,12 +63,14 @@ const (
// Filenames for Constellation's micro services.
//
// ServiceBasePath is the base path for the mounted micro services files.
// ServiceBasePath is the base path for the mounted micro service's files.
ServiceBasePath = "/var/config"
// MeasurementsFilename is the filename of CC measurements.
MeasurementsFilename = "measurements"
// IDFilename is the filename of Constellation's IDs.
IDFilename = "id"
// K8sVersion is the filename of the mapped k8s-version configMap file.
K8sVersion = "k8s-version"
//
// Cryptographic constants.

13
internal/versions/containerImages.go Normal file
View File

@ -0,0 +1,13 @@
package versions
const (
// Constellation images.
JoinImage = "ghcr.io/edgelesssys/constellation/join-service:v1.3.2-0.20220714151638-d295be31"
AccessManagerImage = "ghcr.io/edgelesssys/constellation/access-manager:v1.3.2-0.20220714151638-d295be31"
KmsImage = "ghcr.io/edgelesssys/constellation/kmsserver:v1.3.2-0.20220714151638-d295be31"
VerificationImage = "ghcr.io/edgelesssys/constellation/verification-service:v1.3.2-0.20220714151638-d295be31"
GcpGuestImage = "ghcr.io/edgelesssys/gcp-guest-agent:latest"
// external images.
ClusterAutoscalerImage = "k8s.gcr.io/autoscaling/cluster-autoscaler:v1.23.0"
)

31
internal/versions/k8sBinaries.go Normal file
View File

@ -0,0 +1,31 @@
package versions
// versionConfigs holds download URLs for all required kubernetes components for every supported version.
var VersionConfigs map[string]KubernetesVersion = map[string]KubernetesVersion{
"1.23.6": {
CNIPluginsURL: "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz",
CrictlURL: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.1/crictl-v1.24.1-linux-amd64.tar.gz",
KubeletServiceURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service",
KubeadmConfURL: "https://raw.githubusercontent.com/kubernetes/release/v0.13.0/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf",
KubeletURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubelet",
KubeadmURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubeadm",
KubectlURL: "https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl",
},
}
type KubernetesVersion struct {
CNIPluginsURL string
CrictlURL string
KubeletServiceURL string
KubeadmConfURL string
KubeletURL string
KubeadmURL string
KubectlURL string
}
func IsSupportedK8sVersion(version string) bool {
if _, ok := VersionConfigs[version]; !ok {
return false
}
return true
}

23
joinservice/internal/server/server.go
View File

@ -12,6 +12,7 @@ import (
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/grpc/grpclog"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/internal/versions"
"github.com/edgelesssys/constellation/joinservice/joinproto"
"go.uber.org/zap"
"google.golang.org/grpc"
@ -92,6 +93,12 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi
return nil, status.Errorf(codes.Internal, "unable to generate Kubernetes join arguments: %s", err)
}
log.Infof("Querying K8sVersion ConfigMap")
k8sVersion, err := s.getK8sVersion(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "unable to get k8s version: %s", err)
}
log.Infof("Creating signed kubelet certificate")
kubeletCert, err := s.ca.GetCertificate(req.CertificateRequest)
if err != nil {
@ -125,9 +132,25 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi
DiscoveryTokenCaCertHash: kubeArgs.CACertHashes[0],
KubeletCert: kubeletCert,
ControlPlaneFiles: controlPlaneFiles,
KubernetesVersion: k8sVersion,
}, nil
}
// getK8sVersion reads the k8s version from a VolumeMount that is backed by the k8s-version ConfigMap.
func (s *Server) getK8sVersion(ctx context.Context) (string, error) {
fileContent, err := s.file.Read(filepath.Join(constants.ServiceBasePath, "k8s-version"))
if err != nil {
return "", fmt.Errorf("could not read k8s version file: %v", err)
}
k8sVersion := string(fileContent)
if !versions.IsSupportedK8sVersion(k8sVersion) {
return "", fmt.Errorf("supplied k8s version is not supported: %v", k8sVersion)
}
return k8sVersion, nil
}
// joinTokenGetter returns Kubernetes bootstrap (join) tokens.
type joinTokenGetter interface {
// GetJoinToken returns a bootstrap (join) token.

5
joinservice/internal/server/server_test.go
View File

@ -37,6 +37,7 @@ func TestIssueJoinTicket(t *testing.T) {
CACertHashes: []string{"hash"},
Token: "token",
}
testK8sVersion := "1.23.6"
testCases := map[string]struct {
isControlPlane bool
@ -115,6 +116,10 @@ func TestIssueJoinTicket(t *testing.T) {
if len(tc.id) > 0 {
require.NoError(file.Write(filepath.Join(constants.ServiceBasePath, constants.IDFilename), tc.id, 0o644))
}
// IssueJoinTicket tries to read the k8s-version ConfigMap from a mounted file.
require.NoError(file.Write(filepath.Join(constants.ServiceBasePath, constants.K8sVersion), []byte(testK8sVersion), 0o644))
api := New(
logger.NewTest(t),
file,

43
joinservice/joinproto/join.pb.go
View File

@ -96,6 +96,7 @@ type IssueJoinTicketResponse struct {
Token string `protobuf:"bytes,6,opt,name=token,proto3" json:"token,omitempty"`
DiscoveryTokenCaCertHash string `protobuf:"bytes,7,opt,name=discovery_token_ca_cert_hash,json=discoveryTokenCaCertHash,proto3" json:"discovery_token_ca_cert_hash,omitempty"`
ControlPlaneFiles []*ControlPlaneCertOrKey `protobuf:"bytes,8,rep,name=control_plane_files,json=controlPlaneFiles,proto3" json:"control_plane_files,omitempty"`
KubernetesVersion string `protobuf:"bytes,9,opt,name=kubernetes_version,json=kubernetesVersion,proto3" json:"kubernetes_version,omitempty"`
}
func (x *IssueJoinTicketResponse) Reset() {
@ -186,6 +187,13 @@ func (x *IssueJoinTicketResponse) GetControlPlaneFiles() []*ControlPlaneCertOrKe
return nil
}
func (x *IssueJoinTicketResponse) GetKubernetesVersion() string {
if x != nil {
return x.KubernetesVersion
}
return ""
}
type ControlPlaneCertOrKey struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@ -254,7 +262,7 @@ var file_join_proto_rawDesc = []byte{
0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x69,
0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x18,
0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x69, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c,
0x50, 0x6c, 0x61, 0x6e, 0x65, 0x22, 0xf3, 0x02, 0x0a, 0x17, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a,
0x50, 0x6c, 0x61, 0x6e, 0x65, 0x22, 0xa2, 0x03, 0x0a, 0x17, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a,
0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x64, 0x69, 0x73, 0x6b, 0x5f,
0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x73, 0x74, 0x61, 0x74, 0x65,
@ -277,21 +285,24 @@ var file_join_proto_rawDesc = []byte{
0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x63,
0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x5f, 0x63, 0x65, 0x72,
0x74, 0x5f, 0x6f, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f,
0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x22, 0x43, 0x0a, 0x19, 0x63,
0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x5f, 0x63, 0x65, 0x72,
0x74, 0x5f, 0x6f, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04,
0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61,
0x32, 0x55, 0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, 0x4e, 0x0a, 0x0f, 0x49, 0x73, 0x73, 0x75, 0x65,
0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1c, 0x2e, 0x6a, 0x6f, 0x69,
0x6e, 0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65,
0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e,
0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75,
0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79,
0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f,
0x6a, 0x6f, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x6a, 0x6f, 0x69, 0x6e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x6b,
0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65,
0x74, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x43, 0x0a, 0x19, 0x63, 0x6f,
0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x5f, 0x63, 0x65, 0x72, 0x74,
0x5f, 0x6f, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64,
0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x32,
0x55, 0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, 0x4e, 0x0a, 0x0f, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a,
0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1c, 0x2e, 0x6a, 0x6f, 0x69, 0x6e,
0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x2e, 0x49,
0x73, 0x73, 0x75, 0x65, 0x4a, 0x6f, 0x69, 0x6e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73,
0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a,
0x6f, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x6a, 0x6f, 0x69, 0x6e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (

1
joinservice/joinproto/join.proto
View File

@ -24,6 +24,7 @@ message IssueJoinTicketResponse {
string token = 6;
string discovery_token_ca_cert_hash = 7;
repeated control_plane_cert_or_key control_plane_files = 8;
string kubernetes_version = 9;
}
message control_plane_cert_or_key {