Create kubernetes join token on demand

Signed-off-by: Malte Poll <mp@edgeless.systems>

coordinator/core/cluster.go

@@ -3,17 +3,19 @@ package core
 import (
 	"context"
 	"strings"
+	"time"
 
 	"github.com/edgelesssys/constellation/coordinator/kubernetes"
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
 	"github.com/edgelesssys/constellation/coordinator/role"
+	"github.com/edgelesssys/constellation/internal/constants"
 	"go.uber.org/zap"
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
 )
 
 // GetK8sJoinArgs returns the args needed by a Node to join the cluster.
 func (c *Core) GetK8sJoinArgs() (*kubeadm.BootstrapTokenDiscovery, error) {
-	return c.data().GetKubernetesJoinArgs()
+	return c.kube.GetJoinToken(constants.KubernetesJoinTokenTTL)
 }
 
 // GetK8SCertificateKey returns the key needed by a Coordinator to join the cluster.
@@ -71,7 +73,7 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountUR
 	}
 
 	c.zaplogger.Info("Initializing cluster")
-	joinCommand, err := c.kube.InitCluster(kubernetes.InitClusterInput{
+	if err := c.kube.InitCluster(kubernetes.InitClusterInput{
 		APIServerAdvertiseIP:               coordinatorVPNIP.String(),
 		NodeIP:                             nodeIP,
 		NodeName:                           k8sCompliantHostname(nodeName),
@@ -97,17 +99,11 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountUR
 		CloudNodeManagerImage:              c.cloudNodeManager.Image(),
 		CloudNodeManagerPath:               c.cloudNodeManager.Path(),
 		CloudNodeManagerExtraArgs:          c.cloudNodeManager.ExtraArgs(),
-	})
+	}); err != nil {
-	if err != nil {
 		c.zaplogger.Error("Initializing cluster failed", zap.Error(err))
 		return nil, err
 	}
 
-	if err := c.data().PutKubernetesJoinArgs(joinCommand); err != nil {
-		c.zaplogger.Error("Storing Kubernetes join command failed", zap.Error(err))
-		return nil, err
-	}
-
 	kubeconfig, err := c.kube.GetKubeconfig()
 	if err != nil {
 		return nil, err
@@ -180,25 +176,23 @@ func (c *Core) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, certKey string
 // Cluster manages the overall cluster lifecycle (init, join).
 type Cluster interface {
 	// InitCluster bootstraps a new cluster with the current node being the master, returning the arguments required to join the cluster.
-	InitCluster(kubernetes.InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error)
+	InitCluster(kubernetes.InitClusterInput) error
 	// JoinCluster will join the current node to an existing cluster.
 	JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, ccmSupported bool, peerRole role.Role) error
 	// GetKubeconfig reads the kubeconfig from the filesystem. Only succeeds after cluster is initialized.
 	GetKubeconfig() ([]byte, error)
 	// GetKubeadmCertificateKey returns the 64-byte hex string key needed to join the cluster as control-plane. This function must be executed on a control-plane.
 	GetKubeadmCertificateKey() (string, error)
+	// GetJoinToken returns a bootstrap (join) token.
+	GetJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error)
 }
 
 // ClusterFake behaves like a real cluster, but does not actually initialize or join Kubernetes.
 type ClusterFake struct{}
 
 // InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
-func (c *ClusterFake) InitCluster(kubernetes.InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error) {
+func (c *ClusterFake) InitCluster(kubernetes.InitClusterInput) error {
-	return &kubeadm.BootstrapTokenDiscovery{
+	return nil
-		APIServerEndpoint: "0.0.0.0",
-		Token:             "kube-fake-token",
-		CACertHashes:      []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"},
-	}, nil
 }
 
 // JoinCluster will fake joining the current node to an existing cluster.
@@ -216,6 +210,15 @@ func (c *ClusterFake) GetKubeadmCertificateKey() (string, error) {
 	return "controlPlaneCertficateKey", nil
 }
 
+// GetJoinToken returns a bootstrap (join) token.
+func (c *ClusterFake) GetJoinToken(_ time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) {
+	return &kubeadm.BootstrapTokenDiscovery{
+		APIServerEndpoint: "0.0.0.0",
+		Token:             "kube-fake-token",
+		CACertHashes:      []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"},
+	}, nil
+}
+
 // k8sCompliantHostname transforms a hostname to an RFC 1123 compliant, lowercase subdomain as required by Kubernetes node names.
 // The following regex is used by k8s for validation: /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/ .
 // Only a simple heuristic is used for now (to lowercase, replace underscores).

coordinator/core/cluster_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"regexp"
 	"testing"
+	"time"
 
 	"github.com/edgelesssys/constellation/cli/file"
 	"github.com/edgelesssys/constellation/coordinator/attestation/simulator"
@@ -339,20 +340,21 @@ func TestK8sCompliantHostname(t *testing.T) {
 }
 
 type clusterStub struct {
-	initJoinArgs     kubeadm.BootstrapTokenDiscovery
 	initErr              error
 	joinErr              error
 	kubeconfig           []byte
 	getKubeconfigErr     error
+	getJoinTokenResponse *kubeadm.BootstrapTokenDiscovery
+	getJoinTokenErr      error
 
 	initInputs      []kubernetes.InitClusterInput
 	joinClusterArgs []joinClusterArgs
 }
 
-func (c *clusterStub) InitCluster(in kubernetes.InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error) {
+func (c *clusterStub) InitCluster(in kubernetes.InitClusterInput) error {
 	c.initInputs = append(c.initInputs, in)
 
-	return &c.initJoinArgs, c.initErr
+	return c.initErr
 }
 
 func (c *clusterStub) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, _ bool, _ role.Role) error {
@@ -374,6 +376,10 @@ func (c *clusterStub) GetKubeadmCertificateKey() (string, error) {
 	return "dummy", nil
 }
 
+func (c *clusterStub) GetJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) {
+	return c.getJoinTokenResponse, c.getJoinTokenErr
+}
+
 type prepareInstanceRequest struct {
 	instance Instance
 	vpnIP    string

coordinator/kubernetes/k8sapi/util.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"os/exec"
 	"regexp"
-	"strings"
+	"time"
 
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
@@ -23,7 +23,7 @@ type Client interface {
 }
 
 type ClusterUtil interface {
-	InitCluster(initConfig []byte) (*kubeadm.BootstrapTokenDiscovery, error)
+	InitCluster(initConfig []byte) error
 	JoinCluster(joinConfig []byte) error
 	SetupPodNetwork(kubectl Client, podNetworkConfiguration resources.Marshaler) error
 	SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler, secrets resources.Marshaler) error
@@ -31,73 +31,32 @@ type ClusterUtil interface {
 	SetupCloudNodeManager(kubectl Client, cloudNodeManagerConfiguration resources.Marshaler) error
 	RestartKubelet() error
 	GetControlPlaneJoinCertificateKey() (string, error)
+	CreateJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error)
 }
 
 type KubernetesUtil struct{}
 
-func (k *KubernetesUtil) InitCluster(initConfig []byte) (*kubeadm.BootstrapTokenDiscovery, error) {
+func (k *KubernetesUtil) InitCluster(initConfig []byte) error {
 	initConfigFile, err := os.CreateTemp("", "kubeadm-init.*.yaml")
 	if err != nil {
-		return nil, fmt.Errorf("failed to create init config file %v: %w", initConfigFile.Name(), err)
+		return fmt.Errorf("failed to create init config file %v: %w", initConfigFile.Name(), err)
 	}
 	defer os.Remove(initConfigFile.Name())
 
 	if _, err := initConfigFile.Write(initConfig); err != nil {
-		return nil, fmt.Errorf("writing kubeadm init yaml config %v failed: %w", initConfigFile.Name(), err)
+		return fmt.Errorf("writing kubeadm init yaml config %v failed: %w", initConfigFile.Name(), err)
 	}
 
 	cmd := exec.Command("kubeadm", "init", "--config", initConfigFile.Name())
-	stdout, err := cmd.Output()
+	_, err = cmd.Output()
 	if err != nil {
 		var exitErr *exec.ExitError
 		if errors.As(err, &exitErr) {
-			return nil, fmt.Errorf("kubeadm init failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
+			return fmt.Errorf("kubeadm init failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
 		}
-		return nil, fmt.Errorf("kubeadm init failed: %w", err)
+		return fmt.Errorf("kubeadm init failed: %w", err)
 	}
-
+	return nil
-	stdoutStr := string(stdout)
-	indexKubeadmJoin := strings.Index(stdoutStr, "kubeadm join")
-	if indexKubeadmJoin < 0 {
-		return nil, errors.New("kubeadm init did not return join command")
-	}
-
-	joinCommand := strings.ReplaceAll(stdoutStr[indexKubeadmJoin:], "\\\n", " ")
-	// `kubeadm init` returns the two join commands, each broken up into two lines with backslash + newline in between.
-	// The following functions assume that stdoutStr[indexKubeadmJoin:] look like the following string.
-
-	// -----------------------------------------------------------------------------------------------
-	// --- When modifying the kubeadm.InitConfiguration make sure that this assumption still holds ---
-	// -----------------------------------------------------------------------------------------------
-
-	// "kubeadm join 127.0.0.1:16443 --token vlhjr4.9l6lhek0b9v65m67 \
-	//	--discovery-token-ca-cert-hash sha256:2b5343a162e31b70602e3cab3d87189dc10431e869633c4db63c3bfcd038dee6 \
-	//	--control-plane
-	//
-	// Then you can join any number of worker nodes by running the following on each as root:
-	//
-	// kubeadm join 127.0.0.1:16443 --token vlhjr4.9l6lhek0b9v65m67 \
-	//  --discovery-token-ca-cert-hash sha256:2b5343a162e31b70602e3cab3d87189dc10431e869633c4db63c3bfcd038dee6"
-
-	// Splits the string into a slice, where earch slice-element contains one line from the previous string
-	splittedJoinCommand := strings.SplitN(joinCommand, "\n", 2)
-	joinConfig, err := ParseJoinCommand(splittedJoinCommand[0])
-	if err != nil {
-		return nil, err
-	}
-
-	// create extra join token without expiration
-	cmd = exec.Command("kubeadm", "token", "create", "--ttl", "0")
-	joinToken, err := cmd.Output()
-	if err != nil {
-		var exitErr *exec.ExitError
-		if errors.As(err, &exitErr) {
-			return nil, fmt.Errorf("kubeadm token create failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
-		}
-		return nil, fmt.Errorf("kubeadm token create failed: %w", err)
-	}
-	joinConfig.Token = strings.TrimSpace(string(joinToken))
-	return joinConfig, nil
 }
 
 // SetupPodNetwork sets up the flannel pod network.
@@ -195,3 +154,14 @@ func (k *KubernetesUtil) GetControlPlaneJoinCertificateKey() (string, error) {
 	}
 	return key, nil
 }
+
+// CreateJoinToken creates a new bootstrap (join) token.
+func (k *KubernetesUtil) CreateJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) {
+	output, err := exec.Command("kubeadm", "token", "create", "--ttl", ttl.String(), "--print-join-command").Output()
+	if err != nil {
+		return nil, fmt.Errorf("kubeadm token create failed: %w", err)
+	}
+	// `kubeadm token create [...] --print-join-command` outputs the following format:
+	// kubeadm join [API_SERVER_ENDPOINT] --token [TOKEN] --discovery-token-ca-cert-hash [DISCOVERY_TOKEN_CA_CERT_HASH]
+	return ParseJoinCommand(string(output))
+}

coordinator/kubernetes/kubernetes.go

@@ -3,6 +3,7 @@ package kubernetes
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi"
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
@@ -47,7 +48,7 @@ func New(clusterUtil k8sapi.ClusterUtil, configProvider configurationProvider, c
 }
 
 // InitCluster initializes a new Kubernetes cluster and applies pod network provider.
-func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error) {
+func (k *KubeWrapper) InitCluster(in InitClusterInput) error {
 	initConfig := k.configProvider.InitConfiguration(in.SupportsCloudControllerManager)
 	initConfig.SetApiServerAdvertiseAddress(in.APIServerAdvertiseIP)
 	initConfig.SetNodeIP(in.NodeIP)
@@ -57,20 +58,19 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
 	initConfig.SetProviderID(in.ProviderID)
 	initConfigYAML, err := initConfig.Marshal()
 	if err != nil {
-		return nil, fmt.Errorf("encoding kubeadm init configuration as YAML failed: %w", err)
+		return fmt.Errorf("encoding kubeadm init configuration as YAML failed: %w", err)
 	}
-	joinK8SClusterRequest, err := k.clusterUtil.InitCluster(initConfigYAML)
+	if err := k.clusterUtil.InitCluster(initConfigYAML); err != nil {
-	if err != nil {
+		return fmt.Errorf("kubeadm init failed: %w", err)
-		return nil, fmt.Errorf("kubeadm init failed: %w", err)
 	}
 	kubeConfig, err := k.GetKubeconfig()
 	if err != nil {
-		return nil, fmt.Errorf("reading kubeconfig after cluster initialization failed: %w", err)
+		return fmt.Errorf("reading kubeconfig after cluster initialization failed: %w", err)
 	}
 	k.client.SetKubeconfig(kubeConfig)
 	flannel := resources.NewDefaultFlannelDeployment()
 	if err = k.clusterUtil.SetupPodNetwork(k.client, flannel); err != nil {
-		return nil, fmt.Errorf("setup of pod network failed: %w", err)
+		return fmt.Errorf("setup of pod network failed: %w", err)
 	}
 
 	if in.SupportsCloudControllerManager {
@@ -79,7 +79,7 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
 			in.CloudControllerManagerVolumes, in.CloudControllerManagerVolumeMounts, in.CloudControllerManagerEnv,
 		)
 		if err := k.clusterUtil.SetupCloudControllerManager(k.client, cloudControllerManagerConfiguration, in.CloudControllerManagerConfigMaps, in.CloudControllerManagerSecrets); err != nil {
-			return nil, fmt.Errorf("failed to setup cloud-controller-manager: %w", err)
+			return fmt.Errorf("failed to setup cloud-controller-manager: %w", err)
 		}
 	}
 
@@ -88,7 +88,7 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
 			in.CloudNodeManagerImage, in.CloudNodeManagerPath, in.CloudNodeManagerExtraArgs,
 		)
 		if err := k.clusterUtil.SetupCloudNodeManager(k.client, cloudNodeManagerConfiguration); err != nil {
-			return nil, fmt.Errorf("failed to setup cloud-node-manager: %w", err)
+			return fmt.Errorf("failed to setup cloud-node-manager: %w", err)
 		}
 	}
 
@@ -96,11 +96,11 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
 		clusterAutoscalerConfiguration := resources.NewDefaultAutoscalerDeployment(in.AutoscalingVolumes, in.AutoscalingVolumeMounts, in.AutoscalingEnv)
 		clusterAutoscalerConfiguration.SetAutoscalerCommand(in.AutoscalingCloudprovider, in.AutoscalingNodeGroups)
 		if err := k.clusterUtil.SetupAutoscaling(k.client, clusterAutoscalerConfiguration, in.AutoscalingSecrets); err != nil {
-			return nil, fmt.Errorf("failed to setup cluster-autoscaler: %w", err)
+			return fmt.Errorf("failed to setup cluster-autoscaler: %w", err)
 		}
 	}
 
-	return joinK8SClusterRequest, nil
+	return nil
 }
 
 // JoinCluster joins existing Kubernetes cluster.
@@ -144,6 +144,11 @@ func (k *KubeWrapper) GetKubeadmCertificateKey() (string, error) {
 	return k.clusterUtil.GetControlPlaneJoinCertificateKey()
 }
 
+// GetJoinToken returns a bootstrap (join) token.
+func (k *KubeWrapper) GetJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) {
+	return k.clusterUtil.CreateJoinToken(ttl)
+}
+
 type fakeK8SClient struct {
 	kubeconfig []byte
 }

coordinator/kubernetes/kubernetes_test.go

@@ -3,6 +3,7 @@ package kubernetes
 import (
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi"
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
@@ -19,7 +20,6 @@ func TestMain(m *testing.M) {
 }
 
 type stubClusterUtil struct {
-	joinClusterRequest               *kubeadm.BootstrapTokenDiscovery
 	initClusterErr                   error
 	setupPodNetworkErr               error
 	setupAutoscalingError            error
@@ -27,14 +27,16 @@ type stubClusterUtil struct {
 	setupCloudNodeManagerError       error
 	joinClusterErr                   error
 	restartKubeletErr                error
+	createJoinTokenResponse          *kubeadm.BootstrapTokenDiscovery
+	createJoinTokenErr               error
 
 	initConfigs [][]byte
 	joinConfigs [][]byte
 }
 
-func (s *stubClusterUtil) InitCluster(initConfig []byte) (*kubeadm.BootstrapTokenDiscovery, error) {
+func (s *stubClusterUtil) InitCluster(initConfig []byte) error {
 	s.initConfigs = append(s.initConfigs, initConfig)
-	return s.joinClusterRequest, s.initClusterErr
+	return s.initClusterErr
 }
 
 func (s *stubClusterUtil) SetupPodNetwork(kubectl k8sapi.Client, podNetworkConfiguration resources.Marshaler) error {
@@ -66,6 +68,10 @@ func (s *stubClusterUtil) GetControlPlaneJoinCertificateKey() (string, error) {
 	return "", nil
 }
 
+func (s *stubClusterUtil) CreateJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) {
+	return s.createJoinTokenResponse, s.createJoinTokenErr
+}
+
 type stubConfigProvider struct {
 	InitConfig k8sapi.KubeadmInitYAML
 	JoinConfig k8sapi.KubeadmJoinYAML
@@ -131,33 +137,21 @@ func TestInitCluster(t *testing.T) {
 		wantErr          bool
 	}{
 		"kubeadm init works": {
-			clusterUtil: stubClusterUtil{
+			clusterUtil: stubClusterUtil{},
-				joinClusterRequest: &kubeadm.BootstrapTokenDiscovery{
-					APIServerEndpoint: "192.0.2.0",
-					Token:             "kube-fake-token",
-					CACertHashes:      []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"},
-				},
-			},
 			kubeconfigReader: stubKubeconfigReader{
 				Kubeconfig: []byte("someKubeconfig"),
 			},
 			wantErr: false,
 		},
 		"kubeadm init errors": {
-			clusterUtil: stubClusterUtil{
+			clusterUtil: stubClusterUtil{initClusterErr: someErr},
-				joinClusterRequest: nil,
-				initClusterErr:     someErr,
-			},
 			kubeconfigReader: stubKubeconfigReader{
 				Kubeconfig: []byte("someKubeconfig"),
 			},
 			wantErr: true,
 		},
 		"pod network setup errors": {
-			clusterUtil: stubClusterUtil{
+			clusterUtil: stubClusterUtil{setupPodNetworkErr: someErr},
-				joinClusterRequest: nil,
-				setupPodNetworkErr: someErr,
-			},
 			kubeconfigReader: stubKubeconfigReader{
 				Kubeconfig: []byte("someKubeconfig"),
 			},
@@ -176,7 +170,7 @@ func TestInitCluster(t *testing.T) {
 				client:           &tc.kubeCTL,
 				kubeconfigReader: &tc.kubeconfigReader,
 			}
-			joinCommand, err := kube.InitCluster(
+			err := kube.InitCluster(
 				InitClusterInput{
 					APIServerAdvertiseIP:        coordinatorVPNIP,
 					NodeName:                    instanceName,
@@ -195,7 +189,6 @@ func TestInitCluster(t *testing.T) {
 				return
 			}
 			require.NoError(err)
-			assert.Equal(tc.clusterUtil.joinClusterRequest, joinCommand)
 
 			var kubeadmConfig k8sapi.KubeadmInitYAML
 			require.NoError(resources.UnmarshalK8SResources(tc.clusterUtil.initConfigs[0], &kubeadmConfig))

coordinator/storewrapper/storewrapper.go

@@ -156,15 +156,6 @@ func (s StoreWrapper) GetKubernetesJoinArgs() (*kubeadm.BootstrapTokenDiscovery,
 	return &joinCommand, nil
 }
 
-// PutKubernetesJoinArgs saves the Kubernetes join command to store.
-func (s StoreWrapper) PutKubernetesJoinArgs(args *kubeadm.BootstrapTokenDiscovery) error {
-	j, err := json.Marshal(args)
-	if err != nil {
-		return err
-	}
-	return s.Store.Put(keyKubernetesJoinCommand, j)
-}
-
 // GetKubernetesConfig returns the Kubernetes kubeconfig file to authenticate with the Kubernetes API.
 func (s StoreWrapper) GetKubernetesConfig() ([]byte, error) {
 	return s.Store.Get(keyKubeConfig)

internal/constants/constants.go

@@ -4,6 +4,8 @@ Constants should never be overwritable by command line flags or configuration fi
 */
 package constants
 
+import "time"
+
 const (
 	//
 	// Ports.
@@ -46,6 +48,7 @@ const (
 
 	// KubernetesVersion installed by kubeadm.
 	KubernetesVersion = "stable-1.23"
+	KubernetesJoinTokenTTL = 15 * time.Minute
 )
 
 // CliVersion is the version of the CLI. Left as a separate variable to allow override during build.