Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-10-01 01:36:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

Create kubernetes join token on demand

Browse Source 
Signed-off-by: Malte Poll <mp@edgeless.systems>
This commit is contained in:
Malte Poll 2022-05-04 14:32:34 +02:00 committed by Malte Poll
parent ddcb4dc95f
commit c9226de9ab
7 changed files with 85 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
coordinator/core/cluster.go
View File

@ -3,17 +3,19 @@ package core
import (
"context"
"strings"
"time"
"github.com/edgelesssys/constellation/coordinator/kubernetes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/edgelesssys/constellation/internal/constants"
"go.uber.org/zap"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
// GetK8sJoinArgs returns the args needed by a Node to join the cluster.
func (c *Core) GetK8sJoinArgs() (*kubeadm.BootstrapTokenDiscovery, error) {
return c.data().GetKubernetesJoinArgs()
return c.kube.GetJoinToken(constants.KubernetesJoinTokenTTL)
}
// GetK8SCertificateKey returns the key needed by a Coordinator to join the cluster.
@ -71,7 +73,7 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountUR
}
c.zaplogger.Info("Initializing cluster")
joinCommand, err := c.kube.InitCluster(kubernetes.InitClusterInput{
if err := c.kube.InitCluster(kubernetes.InitClusterInput{
APIServerAdvertiseIP: coordinatorVPNIP.String(),
NodeIP: nodeIP,
NodeName: k8sCompliantHostname(nodeName),
@ -97,17 +99,11 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountUR
CloudNodeManagerImage: c.cloudNodeManager.Image(),
CloudNodeManagerPath: c.cloudNodeManager.Path(),
CloudNodeManagerExtraArgs: c.cloudNodeManager.ExtraArgs(),
})
if err != nil {
}); err != nil {
c.zaplogger.Error("Initializing cluster failed", zap.Error(err))
return nil, err
}
if err := c.data().PutKubernetesJoinArgs(joinCommand); err != nil {
c.zaplogger.Error("Storing Kubernetes join command failed", zap.Error(err))
return nil, err
}
kubeconfig, err := c.kube.GetKubeconfig()
if err != nil {
return nil, err
@ -180,25 +176,23 @@ func (c *Core) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, certKey string
// Cluster manages the overall cluster lifecycle (init, join).
type Cluster interface {
// InitCluster bootstraps a new cluster with the current node being the master, returning the arguments required to join the cluster.
InitCluster(kubernetes.InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error)
InitCluster(kubernetes.InitClusterInput) error
// JoinCluster will join the current node to an existing cluster.
JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, ccmSupported bool, peerRole role.Role) error
// GetKubeconfig reads the kubeconfig from the filesystem. Only succeeds after cluster is initialized.
GetKubeconfig() ([]byte, error)
// GetKubeadmCertificateKey returns the 64-byte hex string key needed to join the cluster as control-plane. This function must be executed on a control-plane.
GetKubeadmCertificateKey() (string, error)
// GetJoinToken returns a bootstrap (join) token.
GetJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error)
}
// ClusterFake behaves like a real cluster, but does not actually initialize or join Kubernetes.
type ClusterFake struct{}
// InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
func (c *ClusterFake) InitCluster(kubernetes.InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error) {
return &kubeadm.BootstrapTokenDiscovery{
APIServerEndpoint: "0.0.0.0",
Token: "kube-fake-token",
CACertHashes: []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"},
}, nil
func (c *ClusterFake) InitCluster(kubernetes.InitClusterInput) error {
return nil
}
// JoinCluster will fake joining the current node to an existing cluster.
@ -216,6 +210,15 @@ func (c *ClusterFake) GetKubeadmCertificateKey() (string, error) {
return "controlPlaneCertficateKey", nil
}
// GetJoinToken returns a bootstrap (join) token.
func (c *ClusterFake) GetJoinToken(_ time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) {
return &kubeadm.BootstrapTokenDiscovery{
APIServerEndpoint: "0.0.0.0",
Token: "kube-fake-token",
CACertHashes: []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"},
}, nil
}
// k8sCompliantHostname transforms a hostname to an RFC 1123 compliant, lowercase subdomain as required by Kubernetes node names.
// The following regex is used by k8s for validation: /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/ .
// Only a simple heuristic is used for now (to lowercase, replace underscores).

12
coordinator/core/cluster_test.go
View File

@ -4,6 +4,7 @@ import (
"errors"
"regexp"
"testing"
"time"
"github.com/edgelesssys/constellation/cli/file"
"github.com/edgelesssys/constellation/coordinator/attestation/simulator"
@ -339,20 +340,21 @@ func TestK8sCompliantHostname(t *testing.T) {
}
type clusterStub struct {
initJoinArgs kubeadm.BootstrapTokenDiscovery
initErr error
joinErr error
kubeconfig []byte
getKubeconfigErr error
getJoinTokenResponse *kubeadm.BootstrapTokenDiscovery
getJoinTokenErr error
initInputs []kubernetes.InitClusterInput
joinClusterArgs []joinClusterArgs
}
func (c *clusterStub) InitCluster(in kubernetes.InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error) {
func (c *clusterStub) InitCluster(in kubernetes.InitClusterInput) error {
c.initInputs = append(c.initInputs, in)
return &c.initJoinArgs, c.initErr
return c.initErr
}
func (c *clusterStub) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, _ bool, _ role.Role) error {
@ -374,6 +376,10 @@ func (c *clusterStub) GetKubeadmCertificateKey() (string, error) {
return "dummy", nil
}
func (c *clusterStub) GetJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) {
return c.getJoinTokenResponse, c.getJoinTokenErr
}
type prepareInstanceRequest struct {
instance Instance
vpnIP string

72
coordinator/kubernetes/k8sapi/util.go
View File

@ -6,7 +6,7 @@ import (
"os"
"os/exec"
"regexp"
"strings"
"time"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
@ -23,7 +23,7 @@ type Client interface {
}
type ClusterUtil interface {
InitCluster(initConfig []byte) (*kubeadm.BootstrapTokenDiscovery, error)
InitCluster(initConfig []byte) error
JoinCluster(joinConfig []byte) error
SetupPodNetwork(kubectl Client, podNetworkConfiguration resources.Marshaler) error
SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler, secrets resources.Marshaler) error
@ -31,73 +31,32 @@ type ClusterUtil interface {
SetupCloudNodeManager(kubectl Client, cloudNodeManagerConfiguration resources.Marshaler) error
RestartKubelet() error
GetControlPlaneJoinCertificateKey() (string, error)
CreateJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error)
}
type KubernetesUtil struct{}
func (k *KubernetesUtil) InitCluster(initConfig []byte) (*kubeadm.BootstrapTokenDiscovery, error) {
func (k *KubernetesUtil) InitCluster(initConfig []byte) error {
initConfigFile, err := os.CreateTemp("", "kubeadm-init.*.yaml")
if err != nil {
return nil, fmt.Errorf("failed to create init config file %v: %w", initConfigFile.Name(), err)
return fmt.Errorf("failed to create init config file %v: %w", initConfigFile.Name(), err)
}
defer os.Remove(initConfigFile.Name())
if _, err := initConfigFile.Write(initConfig); err != nil {
return nil, fmt.Errorf("writing kubeadm init yaml config %v failed: %w", initConfigFile.Name(), err)
return fmt.Errorf("writing kubeadm init yaml config %v failed: %w", initConfigFile.Name(), err)
}
cmd := exec.Command("kubeadm", "init", "--config", initConfigFile.Name())
stdout, err := cmd.Output()
_, err = cmd.Output()
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return nil, fmt.Errorf("kubeadm init failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
return fmt.Errorf("kubeadm init failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
}
return nil, fmt.Errorf("kubeadm init failed: %w", err)
return fmt.Errorf("kubeadm init failed: %w", err)
}
stdoutStr := string(stdout)
indexKubeadmJoin := strings.Index(stdoutStr, "kubeadm join")
if indexKubeadmJoin < 0 {
return nil, errors.New("kubeadm init did not return join command")
}
joinCommand := strings.ReplaceAll(stdoutStr[indexKubeadmJoin:], "\\\n", " ")
// `kubeadm init` returns the two join commands, each broken up into two lines with backslash + newline in between.
// The following functions assume that stdoutStr[indexKubeadmJoin:] look like the following string.
// -----------------------------------------------------------------------------------------------
// --- When modifying the kubeadm.InitConfiguration make sure that this assumption still holds ---
// -----------------------------------------------------------------------------------------------
// "kubeadm join 127.0.0.1:16443 --token vlhjr4.9l6lhek0b9v65m67 \
// --discovery-token-ca-cert-hash sha256:2b5343a162e31b70602e3cab3d87189dc10431e869633c4db63c3bfcd038dee6 \
// --control-plane
//
// Then you can join any number of worker nodes by running the following on each as root:
//
// kubeadm join 127.0.0.1:16443 --token vlhjr4.9l6lhek0b9v65m67 \
// --discovery-token-ca-cert-hash sha256:2b5343a162e31b70602e3cab3d87189dc10431e869633c4db63c3bfcd038dee6"
// Splits the string into a slice, where earch slice-element contains one line from the previous string
splittedJoinCommand := strings.SplitN(joinCommand, "\n", 2)
joinConfig, err := ParseJoinCommand(splittedJoinCommand[0])
if err != nil {
return nil, err
}
// create extra join token without expiration
cmd = exec.Command("kubeadm", "token", "create", "--ttl", "0")
joinToken, err := cmd.Output()
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return nil, fmt.Errorf("kubeadm token create failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
}
return nil, fmt.Errorf("kubeadm token create failed: %w", err)
}
joinConfig.Token = strings.TrimSpace(string(joinToken))
return joinConfig, nil
return nil
}
// SetupPodNetwork sets up the flannel pod network.
@ -195,3 +154,14 @@ func (k *KubernetesUtil) GetControlPlaneJoinCertificateKey() (string, error) {
}
return key, nil
}
// CreateJoinToken creates a new bootstrap (join) token.
func (k *KubernetesUtil) CreateJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) {
output, err := exec.Command("kubeadm", "token", "create", "--ttl", ttl.String(), "--print-join-command").Output()
if err != nil {
return nil, fmt.Errorf("kubeadm token create failed: %w", err)
}
// `kubeadm token create [...] --print-join-command` outputs the following format:
// kubeadm join [API_SERVER_ENDPOINT] --token [TOKEN] --discovery-token-ca-cert-hash [DISCOVERY_TOKEN_CA_CERT_HASH]
return ParseJoinCommand(string(output))
}

27
coordinator/kubernetes/kubernetes.go
View File

@ -3,6 +3,7 @@ package kubernetes
import (
"fmt"
"strings"
"time"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
@ -47,7 +48,7 @@ func New(clusterUtil k8sapi.ClusterUtil, configProvider configurationProvider, c
}
// InitCluster initializes a new Kubernetes cluster and applies pod network provider.
func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error) {
func (k *KubeWrapper) InitCluster(in InitClusterInput) error {
initConfig := k.configProvider.InitConfiguration(in.SupportsCloudControllerManager)
initConfig.SetApiServerAdvertiseAddress(in.APIServerAdvertiseIP)
initConfig.SetNodeIP(in.NodeIP)
@ -57,20 +58,19 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
initConfig.SetProviderID(in.ProviderID)
initConfigYAML, err := initConfig.Marshal()
if err != nil {
return nil, fmt.Errorf("encoding kubeadm init configuration as YAML failed: %w", err)
return fmt.Errorf("encoding kubeadm init configuration as YAML failed: %w", err)
}
joinK8SClusterRequest, err := k.clusterUtil.InitCluster(initConfigYAML)
if err != nil {
return nil, fmt.Errorf("kubeadm init failed: %w", err)
if err := k.clusterUtil.InitCluster(initConfigYAML); err != nil {
return fmt.Errorf("kubeadm init failed: %w", err)
}
kubeConfig, err := k.GetKubeconfig()
if err != nil {
return nil, fmt.Errorf("reading kubeconfig after cluster initialization failed: %w", err)
return fmt.Errorf("reading kubeconfig after cluster initialization failed: %w", err)
}
k.client.SetKubeconfig(kubeConfig)
flannel := resources.NewDefaultFlannelDeployment()
if err = k.clusterUtil.SetupPodNetwork(k.client, flannel); err != nil {
return nil, fmt.Errorf("setup of pod network failed: %w", err)
return fmt.Errorf("setup of pod network failed: %w", err)
}
if in.SupportsCloudControllerManager {
@ -79,7 +79,7 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
in.CloudControllerManagerVolumes, in.CloudControllerManagerVolumeMounts, in.CloudControllerManagerEnv,
)
if err := k.clusterUtil.SetupCloudControllerManager(k.client, cloudControllerManagerConfiguration, in.CloudControllerManagerConfigMaps, in.CloudControllerManagerSecrets); err != nil {
return nil, fmt.Errorf("failed to setup cloud-controller-manager: %w", err)
return fmt.Errorf("failed to setup cloud-controller-manager: %w", err)
}
}
@ -88,7 +88,7 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
in.CloudNodeManagerImage, in.CloudNodeManagerPath, in.CloudNodeManagerExtraArgs,
)
if err := k.clusterUtil.SetupCloudNodeManager(k.client, cloudNodeManagerConfiguration); err != nil {
return nil, fmt.Errorf("failed to setup cloud-node-manager: %w", err)
return fmt.Errorf("failed to setup cloud-node-manager: %w", err)
}
}
@ -96,11 +96,11 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
clusterAutoscalerConfiguration := resources.NewDefaultAutoscalerDeployment(in.AutoscalingVolumes, in.AutoscalingVolumeMounts, in.AutoscalingEnv)
clusterAutoscalerConfiguration.SetAutoscalerCommand(in.AutoscalingCloudprovider, in.AutoscalingNodeGroups)
if err := k.clusterUtil.SetupAutoscaling(k.client, clusterAutoscalerConfiguration, in.AutoscalingSecrets); err != nil {
return nil, fmt.Errorf("failed to setup cluster-autoscaler: %w", err)
return fmt.Errorf("failed to setup cluster-autoscaler: %w", err)
}
}
return joinK8SClusterRequest, nil
return nil
}
// JoinCluster joins existing Kubernetes cluster.
@ -144,6 +144,11 @@ func (k *KubeWrapper) GetKubeadmCertificateKey() (string, error) {
return k.clusterUtil.GetControlPlaneJoinCertificateKey()
}
// GetJoinToken returns a bootstrap (join) token.
func (k *KubeWrapper) GetJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) {
return k.clusterUtil.CreateJoinToken(ttl)
}
type fakeK8SClient struct {
kubeconfig []byte
}

33
coordinator/kubernetes/kubernetes_test.go
View File

@ -3,6 +3,7 @@ package kubernetes
import (
"errors"
"testing"
"time"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
@ -19,7 +20,6 @@ func TestMain(m *testing.M) {
}
type stubClusterUtil struct {
joinClusterRequest *kubeadm.BootstrapTokenDiscovery
initClusterErr error
setupPodNetworkErr error
setupAutoscalingError error
@ -27,14 +27,16 @@ type stubClusterUtil struct {
setupCloudNodeManagerError error
joinClusterErr error
restartKubeletErr error
createJoinTokenResponse *kubeadm.BootstrapTokenDiscovery
createJoinTokenErr error
initConfigs [][]byte
joinConfigs [][]byte
}
func (s *stubClusterUtil) InitCluster(initConfig []byte) (*kubeadm.BootstrapTokenDiscovery, error) {
func (s *stubClusterUtil) InitCluster(initConfig []byte) error {
s.initConfigs = append(s.initConfigs, initConfig)
return s.joinClusterRequest, s.initClusterErr
return s.initClusterErr
}
func (s *stubClusterUtil) SetupPodNetwork(kubectl k8sapi.Client, podNetworkConfiguration resources.Marshaler) error {
@ -66,6 +68,10 @@ func (s *stubClusterUtil) GetControlPlaneJoinCertificateKey() (string, error) {
return "", nil
}
func (s *stubClusterUtil) CreateJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) {
return s.createJoinTokenResponse, s.createJoinTokenErr
}
type stubConfigProvider struct {
InitConfig k8sapi.KubeadmInitYAML
JoinConfig k8sapi.KubeadmJoinYAML
@ -131,33 +137,21 @@ func TestInitCluster(t *testing.T) {
wantErr bool
}{
"kubeadm init works": {
clusterUtil: stubClusterUtil{
joinClusterRequest: &kubeadm.BootstrapTokenDiscovery{
APIServerEndpoint: "192.0.2.0",
Token: "kube-fake-token",
CACertHashes: []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"},
},
},
clusterUtil: stubClusterUtil{},
kubeconfigReader: stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
wantErr: false,
},
"kubeadm init errors": {
clusterUtil: stubClusterUtil{
joinClusterRequest: nil,
initClusterErr: someErr,
},
clusterUtil: stubClusterUtil{initClusterErr: someErr},
kubeconfigReader: stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
wantErr: true,
},
"pod network setup errors": {
clusterUtil: stubClusterUtil{
joinClusterRequest: nil,
setupPodNetworkErr: someErr,
},
clusterUtil: stubClusterUtil{setupPodNetworkErr: someErr},
kubeconfigReader: stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
@ -176,7 +170,7 @@ func TestInitCluster(t *testing.T) {
client: &tc.kubeCTL,
kubeconfigReader: &tc.kubeconfigReader,
}
joinCommand, err := kube.InitCluster(
err := kube.InitCluster(
InitClusterInput{
APIServerAdvertiseIP: coordinatorVPNIP,
NodeName: instanceName,
@ -195,7 +189,6 @@ func TestInitCluster(t *testing.T) {
return
}
require.NoError(err)
assert.Equal(tc.clusterUtil.joinClusterRequest, joinCommand)
var kubeadmConfig k8sapi.KubeadmInitYAML
require.NoError(resources.UnmarshalK8SResources(tc.clusterUtil.initConfigs[0], &kubeadmConfig))

9
coordinator/storewrapper/storewrapper.go
View File

@ -156,15 +156,6 @@ func (s StoreWrapper) GetKubernetesJoinArgs() (*kubeadm.BootstrapTokenDiscovery,
return &joinCommand, nil
}
// PutKubernetesJoinArgs saves the Kubernetes join command to store.
func (s StoreWrapper) PutKubernetesJoinArgs(args *kubeadm.BootstrapTokenDiscovery) error {
j, err := json.Marshal(args)
if err != nil {
return err
}
return s.Store.Put(keyKubernetesJoinCommand, j)
}
// GetKubernetesConfig returns the Kubernetes kubeconfig file to authenticate with the Kubernetes API.
func (s StoreWrapper) GetKubernetesConfig() ([]byte, error) {
return s.Store.Get(keyKubeConfig)

3
internal/constants/constants.go
View File

@ -4,6 +4,8 @@ Constants should never be overwritable by command line flags or configuration fi
*/
package constants
import "time"
const (
//
// Ports.
@ -46,6 +48,7 @@ const (
// KubernetesVersion installed by kubeadm.
KubernetesVersion = "stable-1.23"
KubernetesJoinTokenTTL = 15 * time.Minute
)
// CliVersion is the version of the CLI. Left as a separate variable to allow override during build.