kubernetes: support for certKey request / support for control-plane join

Signed-off-by: Benedict Schlueter <bs@edgeless.systems>

coordinator/core/cluster.go

@@ -16,6 +16,11 @@ func (c *Core) GetK8sJoinArgs() (*kubeadm.BootstrapTokenDiscovery, error) {
 	return c.data().GetKubernetesJoinArgs()
 }
 
+// GetK8SCertificateKey returns the key needed by a Coordinator to join the cluster.
+func (c *Core) GetK8SCertificateKey() (string, error) {
+	return c.kube.GetKubeadmCertificateKey()
+}
+
 // InitCluster initializes the cluster, stores the join args, and returns the kubeconfig.
 func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountURI string) ([]byte, error) {
 	var nodeName string
@@ -123,7 +128,7 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountUR
 }
 
 // JoinCluster lets a Node join the cluster.
-func (c *Core) JoinCluster(args kubeadm.BootstrapTokenDiscovery) error {
+func (c *Core) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, certKey string, peerRole role.Role) error {
 	c.zaplogger.Info("Joining kubernetes cluster")
 	nodeVPNIP, err := c.vpn.GetInterfaceIP()
 	if err != nil {
@@ -155,14 +160,16 @@ func (c *Core) JoinCluster(args kubeadm.BootstrapTokenDiscovery) error {
 		}
 	}
 
-	if err := c.kube.JoinCluster(&args, k8sCompliantHostname(nodeName), nodeIP, providerID); err != nil {
+	c.zaplogger.Info("k8s Join data", zap.String("nodename", nodeName), zap.String("nodeIP", nodeIP), zap.String("nodeVPNIP", nodeVPNIP), zap.String("provid", providerID))
+	// we need to pass the VPNIP for another control-plane, otherwise etcd will bind itself to the wrong IP address and fails
+	if err := c.kube.JoinCluster(args, k8sCompliantHostname(nodeName), nodeIP, nodeVPNIP, providerID, certKey, peerRole); err != nil {
 		c.zaplogger.Error("Joining kubernetes cluster failed", zap.Error(err))
 		return err
 	}
 	c.zaplogger.Info("Joined kubernetes cluster")
 	// set role in cloud provider metadata for autoconfiguration
 	if c.metadata.Supported() {
-		if err := c.metadata.SignalRole(context.TODO(), role.Node); err != nil {
+		if err := c.metadata.SignalRole(context.TODO(), peerRole); err != nil {
 			c.zaplogger.Info("unable to update role in cloud provider metadata", zap.Error(err))
 		}
 	}
@@ -175,9 +182,11 @@ type Cluster interface {
 	// InitCluster bootstraps a new cluster with the current node being the master, returning the arguments required to join the cluster.
 	InitCluster(kubernetes.InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error)
 	// JoinCluster will join the current node to an existing cluster.
-	JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, providerID string) error
+	JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, peerRole role.Role) error
 	// GetKubeconfig reads the kubeconfig from the filesystem. Only succeeds after cluster is initialized.
 	GetKubeconfig() ([]byte, error)
+	// GetKubeadmCertificateKey returns the 64-byte hex string key needed to join the cluster as control-plane. This function must be executed on a control-plane.
+	GetKubeadmCertificateKey() (string, error)
 }
 
 // ClusterFake behaves like a real cluster, but does not actually initialize or join kubernetes.
@@ -193,7 +202,7 @@ func (c *ClusterFake) InitCluster(kubernetes.InitClusterInput) (*kubeadm.Bootstr
 }
 
 // JoinCluster will fake joining the current node to an existing cluster.
-func (c *ClusterFake) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, providerID string) error {
+func (c *ClusterFake) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, _ role.Role) error {
 	return nil
 }
 
@@ -202,6 +211,11 @@ func (c *ClusterFake) GetKubeconfig() ([]byte, error) {
 	return []byte("kubeconfig"), nil
 }
 
+// GetKubeadmCertificateKey fakes generating a certificateKey.
+func (c *ClusterFake) GetKubeadmCertificateKey() (string, error) {
+	return "controlPlaneCertficateKey", nil
+}
+
 // k8sCompliantHostname transforms a hostname to an RFC 1123 compliant, lowercase subdomain as required by kubernetes node names.
 // The following regex is used by k8s for validation: /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/ .
 // Only a simple heuristic is used for now (to lowercase, replace underscores).

coordinator/core/cluster_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/edgelesssys/constellation/coordinator/attestation/simulator"
 	"github.com/edgelesssys/constellation/coordinator/kubernetes"
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
+	"github.com/edgelesssys/constellation/coordinator/role"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -287,12 +288,12 @@ func TestJoinCluster(t *testing.T) {
 			core, err := NewCore(&tc.vpn, &tc.cluster, &tc.metadata, &tc.cloudControllerManager, &tc.cloudNodeManager, &tc.clusterAutoscaler, nil, zapLogger, simulator.OpenSimulatedTPM, nil, file.NewHandler(afero.NewMemMapFs()))
 			require.NoError(err)
 
-			joinReq := kubeadm.BootstrapTokenDiscovery{
+			joinReq := &kubeadm.BootstrapTokenDiscovery{
 				APIServerEndpoint: "192.0.2.0:6443",
 				Token:             "someToken",
 				CACertHashes:      []string{"someHash"},
 			}
-			err = core.JoinCluster(joinReq)
+			err = core.JoinCluster(joinReq, "", role.Node)
 
 			if tc.expectErr {
 				assert.Error(err)
@@ -354,7 +355,7 @@ func (c *clusterStub) InitCluster(in kubernetes.InitClusterInput) (*kubeadm.Boot
 	return &c.initJoinArgs, c.initErr
 }
 
-func (c *clusterStub) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, providerID string) error {
+func (c *clusterStub) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, _ role.Role) error {
 	c.joinClusterArgs = append(c.joinClusterArgs, joinClusterArgs{
 		args:       args,
 		nodeName:   nodeName,
@@ -369,6 +370,10 @@ func (c *clusterStub) GetKubeconfig() ([]byte, error) {
 	return c.kubeconfig, c.getKubeconfigErr
 }
 
+func (c *clusterStub) GetKubeadmCertificateKey() (string, error) {
+	return "dummy", nil
+}
+
 type prepareInstanceRequest struct {
 	instance Instance
 	vpnIP    string

coordinator/kubernetes/k8sapi/joinargs.go

@@ -32,6 +32,7 @@ func ParseJoinCommand(joinCommand string) (*kubeadm.BootstrapTokenDiscovery, err
 	flags.StringVar(&result.Token, "token", "", "")
 	flags.StringVar(&caCertHash, "discovery-token-ca-cert-hash", "", "")
 	flags.Bool("control-plane", false, "")
+	flags.String("certificate-key", "", "")
 	if err := flags.Parse(argv[3:]); err != nil {
 		return nil, fmt.Errorf("parsing flag arguments failed: %v %w", argv, err)
 	}

coordinator/kubernetes/k8sapi/kubeadm_config.go

@@ -179,6 +179,16 @@ func (k *KubeadmJoinYAML) SetProviderID(providerID string) {
 	k.KubeletConfiguration.ProviderID = providerID
 }
 
+func (k *KubeadmJoinYAML) SetControlPlane(advertiseAddress string, certificateKey string) {
+	k.JoinConfiguration.ControlPlane = &kubeadm.JoinControlPlane{
+		LocalAPIEndpoint: kubeadm.APIEndpoint{
+			AdvertiseAddress: advertiseAddress,
+			BindPort:         6443,
+		},
+		CertificateKey: certificateKey,
+	}
+}
+
 func (k *KubeadmJoinYAML) Marshal() ([]byte, error) {
 	return resources.MarshalK8SResources(k)
 }

coordinator/kubernetes/k8sapi/kubeadm_config_test.go

@@ -103,6 +103,7 @@ func TestJoinConfiguration(t *testing.T) {
 				c.SetToken("token")
 				c.AppendDiscoveryTokenCaCertHash("discovery-token-ca-cert-hash")
 				c.SetProviderID("somecloudprovider://instance-id")
+				c.SetControlPlane("192.0.2.0", "11111111111111111111111111111111111")
 				return c
 			}(),
 		},

coordinator/kubernetes/k8sapi/util.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"regexp"
 	"strings"
 
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
@@ -29,6 +30,7 @@ type ClusterUtil interface {
 	SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error
 	SetupCloudNodeManager(kubectl Client, cloudNodeManagerConfiguration resources.Marshaler) error
 	RestartKubelet() error
+	GetControlPlaneJoinCertificateKey() (string, error)
 }
 
 type KubernetesUtil struct{}
@@ -167,3 +169,29 @@ func (k *KubernetesUtil) JoinCluster(joinConfig []byte) error {
 func (k *KubernetesUtil) RestartKubelet() error {
 	return RestartSystemdUnit("kubelet.service")
 }
+
+// GetControlPlaneJoinCertificateKey return the key which can be used in combination with the joinArgs
+// to join the Cluster as control-plane.
+func (k *KubernetesUtil) GetControlPlaneJoinCertificateKey() (string, error) {
+	// Key will be valid for 1h (no option to reduce the duration).
+	// https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-init-phase/#cmd-phase-upload-certs
+	output, err := exec.Command("kubeadm", "init", "phase", "upload-certs", "--upload-certs").Output()
+	if err != nil {
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			return "", fmt.Errorf("kubeadm upload-certs failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
+		}
+		return "", fmt.Errorf("kubeadm upload-certs failed: %w", err)
+	}
+	// Example output:
+	/*
+		[upload-certs] Storing the certificates in ConfigMap "kubeadm-certs" in the "kube-system" Namespace
+		[upload-certs] Using certificate key:
+		9555b74008f24687eb964bd90a164ecb5760a89481d9c55a77c129b7db438168
+	*/
+	key := regexp.MustCompile("[a-f0-9]{64}").FindString(string(output))
+	if key == "" {
+		return "", fmt.Errorf("failed to parse kubeadm output: %s", string(output))
+	}
+	return key, nil
+}

coordinator/kubernetes/kubernetes.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi"
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
+	"github.com/edgelesssys/constellation/coordinator/role"
 	"github.com/spf13/afero"
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
 )
@@ -103,14 +104,17 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
 }
 
 // JoinCluster joins existing kubernetes cluster.
-func (k *KubeWrapper) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, providerID string) error {
+func (k *KubeWrapper) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeInternalIP, nodeVPNIP, providerID, certKey string, peerRole role.Role) error {
 	joinConfig := k.configProvider.JoinConfiguration()
 	joinConfig.SetApiServerEndpoint(args.APIServerEndpoint)
 	joinConfig.SetToken(args.Token)
 	joinConfig.AppendDiscoveryTokenCaCertHash(args.CACertHashes[0])
-	joinConfig.SetNodeIP(nodeIP)
+	joinConfig.SetNodeIP(nodeInternalIP)
 	joinConfig.SetNodeName(nodeName)
 	joinConfig.SetProviderID(providerID)
+	if peerRole == role.Coordinator {
+		joinConfig.SetControlPlane(nodeVPNIP, certKey)
+	}
 	joinConfigYAML, err := joinConfig.Marshal()
 	if err != nil {
 		return fmt.Errorf("encoding kubeadm join configuration as YAML failed: %w", err)
@@ -135,6 +139,11 @@ func (k *KubeWrapper) GetKubeconfig() ([]byte, error) {
 	return []byte(strings.ReplaceAll(string(kubeconf), "127.0.0.1:16443", "10.118.0.1:6443")), nil
 }
 
+// GetKubeadmCertificateKey return the key needed to join the Cluster as Control-Plane (has to be executed on a control-plane; errors otherwise).
+func (k *KubeWrapper) GetKubeadmCertificateKey() (string, error) {
+	return k.clusterUtil.GetControlPlaneJoinCertificateKey()
+}
+
 type fakeK8SClient struct {
 	kubeconfig []byte
 }

coordinator/kubernetes/kubernetes_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi"
 	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
+	"github.com/edgelesssys/constellation/coordinator/role"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
@@ -64,6 +65,10 @@ func (s *stubClusterUtil) RestartKubelet() error {
 	return s.restartKubeletErr
 }
 
+func (s *stubClusterUtil) GetControlPlaneJoinCertificateKey() (string, error) {
+	return "", nil
+}
+
 type stubConfigProvider struct {
 	InitConfig k8sapi.KubeadmInitYAML
 	JoinConfig k8sapi.KubeadmJoinYAML
@@ -236,7 +241,7 @@ func TestJoinCluster(t *testing.T) {
 			require := require.New(t)
 
 			kube := New(&tc.clusterUtil, &stubConfigProvider{}, &client)
-			err := kube.JoinCluster(joinCommand, instanceName, nodeVPNIP, coordinatorProviderID)
+			err := kube.JoinCluster(joinCommand, instanceName, nodeVPNIP, nodeVPNIP, coordinatorProviderID, "", role.Node)
 			if tc.expectErr {
 				assert.Error(err)
 				return