From c9226de9ab3735fd629ac95850b640310f683d07 Mon Sep 17 00:00:00 2001 From: Malte Poll Date: Wed, 4 May 2022 14:32:34 +0200 Subject: [PATCH] Create kubernetes join token on demand Signed-off-by: Malte Poll --- coordinator/core/cluster.go | 35 ++++++----- coordinator/core/cluster_test.go | 20 ++++--- coordinator/kubernetes/k8sapi/util.go | 72 +++++++---------------- coordinator/kubernetes/kubernetes.go | 27 +++++---- coordinator/kubernetes/kubernetes_test.go | 33 ++++------- coordinator/storewrapper/storewrapper.go | 9 --- internal/constants/constants.go | 3 + 7 files changed, 85 insertions(+), 114 deletions(-) diff --git a/coordinator/core/cluster.go b/coordinator/core/cluster.go index 363fe817f..9ea67c9be 100644 --- a/coordinator/core/cluster.go +++ b/coordinator/core/cluster.go @@ -3,17 +3,19 @@ package core import ( "context" "strings" + "time" "github.com/edgelesssys/constellation/coordinator/kubernetes" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/coordinator/role" + "github.com/edgelesssys/constellation/internal/constants" "go.uber.org/zap" kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" ) // GetK8sJoinArgs returns the args needed by a Node to join the cluster. func (c *Core) GetK8sJoinArgs() (*kubeadm.BootstrapTokenDiscovery, error) { - return c.data().GetKubernetesJoinArgs() + return c.kube.GetJoinToken(constants.KubernetesJoinTokenTTL) } // GetK8SCertificateKey returns the key needed by a Coordinator to join the cluster. @@ -71,7 +73,7 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountUR } c.zaplogger.Info("Initializing cluster") - joinCommand, err := c.kube.InitCluster(kubernetes.InitClusterInput{ + if err := c.kube.InitCluster(kubernetes.InitClusterInput{ APIServerAdvertiseIP: coordinatorVPNIP.String(), NodeIP: nodeIP, NodeName: k8sCompliantHostname(nodeName), @@ -97,17 +99,11 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountUR CloudNodeManagerImage: c.cloudNodeManager.Image(), CloudNodeManagerPath: c.cloudNodeManager.Path(), CloudNodeManagerExtraArgs: c.cloudNodeManager.ExtraArgs(), - }) - if err != nil { + }); err != nil { c.zaplogger.Error("Initializing cluster failed", zap.Error(err)) return nil, err } - if err := c.data().PutKubernetesJoinArgs(joinCommand); err != nil { - c.zaplogger.Error("Storing Kubernetes join command failed", zap.Error(err)) - return nil, err - } - kubeconfig, err := c.kube.GetKubeconfig() if err != nil { return nil, err @@ -180,25 +176,23 @@ func (c *Core) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, certKey string // Cluster manages the overall cluster lifecycle (init, join). type Cluster interface { // InitCluster bootstraps a new cluster with the current node being the master, returning the arguments required to join the cluster. - InitCluster(kubernetes.InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error) + InitCluster(kubernetes.InitClusterInput) error // JoinCluster will join the current node to an existing cluster. JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, ccmSupported bool, peerRole role.Role) error // GetKubeconfig reads the kubeconfig from the filesystem. Only succeeds after cluster is initialized. GetKubeconfig() ([]byte, error) // GetKubeadmCertificateKey returns the 64-byte hex string key needed to join the cluster as control-plane. This function must be executed on a control-plane. GetKubeadmCertificateKey() (string, error) + // GetJoinToken returns a bootstrap (join) token. + GetJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) } // ClusterFake behaves like a real cluster, but does not actually initialize or join Kubernetes. type ClusterFake struct{} // InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster. -func (c *ClusterFake) InitCluster(kubernetes.InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error) { - return &kubeadm.BootstrapTokenDiscovery{ - APIServerEndpoint: "0.0.0.0", - Token: "kube-fake-token", - CACertHashes: []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"}, - }, nil +func (c *ClusterFake) InitCluster(kubernetes.InitClusterInput) error { + return nil } // JoinCluster will fake joining the current node to an existing cluster. @@ -216,6 +210,15 @@ func (c *ClusterFake) GetKubeadmCertificateKey() (string, error) { return "controlPlaneCertficateKey", nil } +// GetJoinToken returns a bootstrap (join) token. +func (c *ClusterFake) GetJoinToken(_ time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) { + return &kubeadm.BootstrapTokenDiscovery{ + APIServerEndpoint: "0.0.0.0", + Token: "kube-fake-token", + CACertHashes: []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"}, + }, nil +} + // k8sCompliantHostname transforms a hostname to an RFC 1123 compliant, lowercase subdomain as required by Kubernetes node names. // The following regex is used by k8s for validation: /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/ . // Only a simple heuristic is used for now (to lowercase, replace underscores). diff --git a/coordinator/core/cluster_test.go b/coordinator/core/cluster_test.go index caac3bf2a..00c5e174c 100644 --- a/coordinator/core/cluster_test.go +++ b/coordinator/core/cluster_test.go @@ -4,6 +4,7 @@ import ( "errors" "regexp" "testing" + "time" "github.com/edgelesssys/constellation/cli/file" "github.com/edgelesssys/constellation/coordinator/attestation/simulator" @@ -339,20 +340,21 @@ func TestK8sCompliantHostname(t *testing.T) { } type clusterStub struct { - initJoinArgs kubeadm.BootstrapTokenDiscovery - initErr error - joinErr error - kubeconfig []byte - getKubeconfigErr error + initErr error + joinErr error + kubeconfig []byte + getKubeconfigErr error + getJoinTokenResponse *kubeadm.BootstrapTokenDiscovery + getJoinTokenErr error initInputs []kubernetes.InitClusterInput joinClusterArgs []joinClusterArgs } -func (c *clusterStub) InitCluster(in kubernetes.InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error) { +func (c *clusterStub) InitCluster(in kubernetes.InitClusterInput) error { c.initInputs = append(c.initInputs, in) - return &c.initJoinArgs, c.initErr + return c.initErr } func (c *clusterStub) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, _ bool, _ role.Role) error { @@ -374,6 +376,10 @@ func (c *clusterStub) GetKubeadmCertificateKey() (string, error) { return "dummy", nil } +func (c *clusterStub) GetJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) { + return c.getJoinTokenResponse, c.getJoinTokenErr +} + type prepareInstanceRequest struct { instance Instance vpnIP string diff --git a/coordinator/kubernetes/k8sapi/util.go b/coordinator/kubernetes/k8sapi/util.go index 764c0f51d..cfcb021ab 100644 --- a/coordinator/kubernetes/k8sapi/util.go +++ b/coordinator/kubernetes/k8sapi/util.go @@ -6,7 +6,7 @@ import ( "os" "os/exec" "regexp" - "strings" + "time" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" @@ -23,7 +23,7 @@ type Client interface { } type ClusterUtil interface { - InitCluster(initConfig []byte) (*kubeadm.BootstrapTokenDiscovery, error) + InitCluster(initConfig []byte) error JoinCluster(joinConfig []byte) error SetupPodNetwork(kubectl Client, podNetworkConfiguration resources.Marshaler) error SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler, secrets resources.Marshaler) error @@ -31,73 +31,32 @@ type ClusterUtil interface { SetupCloudNodeManager(kubectl Client, cloudNodeManagerConfiguration resources.Marshaler) error RestartKubelet() error GetControlPlaneJoinCertificateKey() (string, error) + CreateJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) } type KubernetesUtil struct{} -func (k *KubernetesUtil) InitCluster(initConfig []byte) (*kubeadm.BootstrapTokenDiscovery, error) { +func (k *KubernetesUtil) InitCluster(initConfig []byte) error { initConfigFile, err := os.CreateTemp("", "kubeadm-init.*.yaml") if err != nil { - return nil, fmt.Errorf("failed to create init config file %v: %w", initConfigFile.Name(), err) + return fmt.Errorf("failed to create init config file %v: %w", initConfigFile.Name(), err) } defer os.Remove(initConfigFile.Name()) if _, err := initConfigFile.Write(initConfig); err != nil { - return nil, fmt.Errorf("writing kubeadm init yaml config %v failed: %w", initConfigFile.Name(), err) + return fmt.Errorf("writing kubeadm init yaml config %v failed: %w", initConfigFile.Name(), err) } cmd := exec.Command("kubeadm", "init", "--config", initConfigFile.Name()) - stdout, err := cmd.Output() + _, err = cmd.Output() if err != nil { var exitErr *exec.ExitError if errors.As(err, &exitErr) { - return nil, fmt.Errorf("kubeadm init failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr) + return fmt.Errorf("kubeadm init failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr) } - return nil, fmt.Errorf("kubeadm init failed: %w", err) + return fmt.Errorf("kubeadm init failed: %w", err) } - - stdoutStr := string(stdout) - indexKubeadmJoin := strings.Index(stdoutStr, "kubeadm join") - if indexKubeadmJoin < 0 { - return nil, errors.New("kubeadm init did not return join command") - } - - joinCommand := strings.ReplaceAll(stdoutStr[indexKubeadmJoin:], "\\\n", " ") - // `kubeadm init` returns the two join commands, each broken up into two lines with backslash + newline in between. - // The following functions assume that stdoutStr[indexKubeadmJoin:] look like the following string. - - // ----------------------------------------------------------------------------------------------- - // --- When modifying the kubeadm.InitConfiguration make sure that this assumption still holds --- - // ----------------------------------------------------------------------------------------------- - - // "kubeadm join 127.0.0.1:16443 --token vlhjr4.9l6lhek0b9v65m67 \ - // --discovery-token-ca-cert-hash sha256:2b5343a162e31b70602e3cab3d87189dc10431e869633c4db63c3bfcd038dee6 \ - // --control-plane - // - // Then you can join any number of worker nodes by running the following on each as root: - // - // kubeadm join 127.0.0.1:16443 --token vlhjr4.9l6lhek0b9v65m67 \ - // --discovery-token-ca-cert-hash sha256:2b5343a162e31b70602e3cab3d87189dc10431e869633c4db63c3bfcd038dee6" - - // Splits the string into a slice, where earch slice-element contains one line from the previous string - splittedJoinCommand := strings.SplitN(joinCommand, "\n", 2) - joinConfig, err := ParseJoinCommand(splittedJoinCommand[0]) - if err != nil { - return nil, err - } - - // create extra join token without expiration - cmd = exec.Command("kubeadm", "token", "create", "--ttl", "0") - joinToken, err := cmd.Output() - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return nil, fmt.Errorf("kubeadm token create failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr) - } - return nil, fmt.Errorf("kubeadm token create failed: %w", err) - } - joinConfig.Token = strings.TrimSpace(string(joinToken)) - return joinConfig, nil + return nil } // SetupPodNetwork sets up the flannel pod network. @@ -195,3 +154,14 @@ func (k *KubernetesUtil) GetControlPlaneJoinCertificateKey() (string, error) { } return key, nil } + +// CreateJoinToken creates a new bootstrap (join) token. +func (k *KubernetesUtil) CreateJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) { + output, err := exec.Command("kubeadm", "token", "create", "--ttl", ttl.String(), "--print-join-command").Output() + if err != nil { + return nil, fmt.Errorf("kubeadm token create failed: %w", err) + } + // `kubeadm token create [...] --print-join-command` outputs the following format: + // kubeadm join [API_SERVER_ENDPOINT] --token [TOKEN] --discovery-token-ca-cert-hash [DISCOVERY_TOKEN_CA_CERT_HASH] + return ParseJoinCommand(string(output)) +} diff --git a/coordinator/kubernetes/kubernetes.go b/coordinator/kubernetes/kubernetes.go index 465e58bdc..392d64775 100644 --- a/coordinator/kubernetes/kubernetes.go +++ b/coordinator/kubernetes/kubernetes.go @@ -3,6 +3,7 @@ package kubernetes import ( "fmt" "strings" + "time" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" @@ -47,7 +48,7 @@ func New(clusterUtil k8sapi.ClusterUtil, configProvider configurationProvider, c } // InitCluster initializes a new Kubernetes cluster and applies pod network provider. -func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error) { +func (k *KubeWrapper) InitCluster(in InitClusterInput) error { initConfig := k.configProvider.InitConfiguration(in.SupportsCloudControllerManager) initConfig.SetApiServerAdvertiseAddress(in.APIServerAdvertiseIP) initConfig.SetNodeIP(in.NodeIP) @@ -57,20 +58,19 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD initConfig.SetProviderID(in.ProviderID) initConfigYAML, err := initConfig.Marshal() if err != nil { - return nil, fmt.Errorf("encoding kubeadm init configuration as YAML failed: %w", err) + return fmt.Errorf("encoding kubeadm init configuration as YAML failed: %w", err) } - joinK8SClusterRequest, err := k.clusterUtil.InitCluster(initConfigYAML) - if err != nil { - return nil, fmt.Errorf("kubeadm init failed: %w", err) + if err := k.clusterUtil.InitCluster(initConfigYAML); err != nil { + return fmt.Errorf("kubeadm init failed: %w", err) } kubeConfig, err := k.GetKubeconfig() if err != nil { - return nil, fmt.Errorf("reading kubeconfig after cluster initialization failed: %w", err) + return fmt.Errorf("reading kubeconfig after cluster initialization failed: %w", err) } k.client.SetKubeconfig(kubeConfig) flannel := resources.NewDefaultFlannelDeployment() if err = k.clusterUtil.SetupPodNetwork(k.client, flannel); err != nil { - return nil, fmt.Errorf("setup of pod network failed: %w", err) + return fmt.Errorf("setup of pod network failed: %w", err) } if in.SupportsCloudControllerManager { @@ -79,7 +79,7 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD in.CloudControllerManagerVolumes, in.CloudControllerManagerVolumeMounts, in.CloudControllerManagerEnv, ) if err := k.clusterUtil.SetupCloudControllerManager(k.client, cloudControllerManagerConfiguration, in.CloudControllerManagerConfigMaps, in.CloudControllerManagerSecrets); err != nil { - return nil, fmt.Errorf("failed to setup cloud-controller-manager: %w", err) + return fmt.Errorf("failed to setup cloud-controller-manager: %w", err) } } @@ -88,7 +88,7 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD in.CloudNodeManagerImage, in.CloudNodeManagerPath, in.CloudNodeManagerExtraArgs, ) if err := k.clusterUtil.SetupCloudNodeManager(k.client, cloudNodeManagerConfiguration); err != nil { - return nil, fmt.Errorf("failed to setup cloud-node-manager: %w", err) + return fmt.Errorf("failed to setup cloud-node-manager: %w", err) } } @@ -96,11 +96,11 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD clusterAutoscalerConfiguration := resources.NewDefaultAutoscalerDeployment(in.AutoscalingVolumes, in.AutoscalingVolumeMounts, in.AutoscalingEnv) clusterAutoscalerConfiguration.SetAutoscalerCommand(in.AutoscalingCloudprovider, in.AutoscalingNodeGroups) if err := k.clusterUtil.SetupAutoscaling(k.client, clusterAutoscalerConfiguration, in.AutoscalingSecrets); err != nil { - return nil, fmt.Errorf("failed to setup cluster-autoscaler: %w", err) + return fmt.Errorf("failed to setup cluster-autoscaler: %w", err) } } - return joinK8SClusterRequest, nil + return nil } // JoinCluster joins existing Kubernetes cluster. @@ -144,6 +144,11 @@ func (k *KubeWrapper) GetKubeadmCertificateKey() (string, error) { return k.clusterUtil.GetControlPlaneJoinCertificateKey() } +// GetJoinToken returns a bootstrap (join) token. +func (k *KubeWrapper) GetJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) { + return k.clusterUtil.CreateJoinToken(ttl) +} + type fakeK8SClient struct { kubeconfig []byte } diff --git a/coordinator/kubernetes/kubernetes_test.go b/coordinator/kubernetes/kubernetes_test.go index 4ba2df396..50d3b95cd 100644 --- a/coordinator/kubernetes/kubernetes_test.go +++ b/coordinator/kubernetes/kubernetes_test.go @@ -3,6 +3,7 @@ package kubernetes import ( "errors" "testing" + "time" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi" "github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources" @@ -19,7 +20,6 @@ func TestMain(m *testing.M) { } type stubClusterUtil struct { - joinClusterRequest *kubeadm.BootstrapTokenDiscovery initClusterErr error setupPodNetworkErr error setupAutoscalingError error @@ -27,14 +27,16 @@ type stubClusterUtil struct { setupCloudNodeManagerError error joinClusterErr error restartKubeletErr error + createJoinTokenResponse *kubeadm.BootstrapTokenDiscovery + createJoinTokenErr error initConfigs [][]byte joinConfigs [][]byte } -func (s *stubClusterUtil) InitCluster(initConfig []byte) (*kubeadm.BootstrapTokenDiscovery, error) { +func (s *stubClusterUtil) InitCluster(initConfig []byte) error { s.initConfigs = append(s.initConfigs, initConfig) - return s.joinClusterRequest, s.initClusterErr + return s.initClusterErr } func (s *stubClusterUtil) SetupPodNetwork(kubectl k8sapi.Client, podNetworkConfiguration resources.Marshaler) error { @@ -66,6 +68,10 @@ func (s *stubClusterUtil) GetControlPlaneJoinCertificateKey() (string, error) { return "", nil } +func (s *stubClusterUtil) CreateJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) { + return s.createJoinTokenResponse, s.createJoinTokenErr +} + type stubConfigProvider struct { InitConfig k8sapi.KubeadmInitYAML JoinConfig k8sapi.KubeadmJoinYAML @@ -131,33 +137,21 @@ func TestInitCluster(t *testing.T) { wantErr bool }{ "kubeadm init works": { - clusterUtil: stubClusterUtil{ - joinClusterRequest: &kubeadm.BootstrapTokenDiscovery{ - APIServerEndpoint: "192.0.2.0", - Token: "kube-fake-token", - CACertHashes: []string{"sha256:a60ebe9b0879090edd83b40a4df4bebb20506bac1e51d518ff8f4505a721930f"}, - }, - }, + clusterUtil: stubClusterUtil{}, kubeconfigReader: stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, wantErr: false, }, "kubeadm init errors": { - clusterUtil: stubClusterUtil{ - joinClusterRequest: nil, - initClusterErr: someErr, - }, + clusterUtil: stubClusterUtil{initClusterErr: someErr}, kubeconfigReader: stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, wantErr: true, }, "pod network setup errors": { - clusterUtil: stubClusterUtil{ - joinClusterRequest: nil, - setupPodNetworkErr: someErr, - }, + clusterUtil: stubClusterUtil{setupPodNetworkErr: someErr}, kubeconfigReader: stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, @@ -176,7 +170,7 @@ func TestInitCluster(t *testing.T) { client: &tc.kubeCTL, kubeconfigReader: &tc.kubeconfigReader, } - joinCommand, err := kube.InitCluster( + err := kube.InitCluster( InitClusterInput{ APIServerAdvertiseIP: coordinatorVPNIP, NodeName: instanceName, @@ -195,7 +189,6 @@ func TestInitCluster(t *testing.T) { return } require.NoError(err) - assert.Equal(tc.clusterUtil.joinClusterRequest, joinCommand) var kubeadmConfig k8sapi.KubeadmInitYAML require.NoError(resources.UnmarshalK8SResources(tc.clusterUtil.initConfigs[0], &kubeadmConfig)) diff --git a/coordinator/storewrapper/storewrapper.go b/coordinator/storewrapper/storewrapper.go index 797ca0a48..33a3d9c51 100644 --- a/coordinator/storewrapper/storewrapper.go +++ b/coordinator/storewrapper/storewrapper.go @@ -156,15 +156,6 @@ func (s StoreWrapper) GetKubernetesJoinArgs() (*kubeadm.BootstrapTokenDiscovery, return &joinCommand, nil } -// PutKubernetesJoinArgs saves the Kubernetes join command to store. -func (s StoreWrapper) PutKubernetesJoinArgs(args *kubeadm.BootstrapTokenDiscovery) error { - j, err := json.Marshal(args) - if err != nil { - return err - } - return s.Store.Put(keyKubernetesJoinCommand, j) -} - // GetKubernetesConfig returns the Kubernetes kubeconfig file to authenticate with the Kubernetes API. func (s StoreWrapper) GetKubernetesConfig() ([]byte, error) { return s.Store.Get(keyKubeConfig) diff --git a/internal/constants/constants.go b/internal/constants/constants.go index be145effb..c3f4e4936 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -4,6 +4,8 @@ Constants should never be overwritable by command line flags or configuration fi */ package constants +import "time" + const ( // // Ports. @@ -46,6 +48,7 @@ const ( // KubernetesVersion installed by kubeadm. KubernetesVersion = "stable-1.23" + KubernetesJoinTokenTTL = 15 * time.Minute ) // CliVersion is the version of the CLI. Left as a separate variable to allow override during build.