Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-10-01 01:36:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

kubernetes: support for certKey request / support for control-plane join

Browse Source 
Signed-off-by: Benedict Schlueter <bs@edgeless.systems>
This commit is contained in:
Benedict Schlueter 2022-04-25 17:24:48 +02:00 committed by Benedict Schlüter
parent 49def1e97f
commit 0ac9617dac
8 changed files with 84 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
coordinator/core/cluster.go
View File

@ -16,6 +16,11 @@ func (c *Core) GetK8sJoinArgs() (*kubeadm.BootstrapTokenDiscovery, error) {
return c.data().GetKubernetesJoinArgs()
}
// GetK8SCertificateKey returns the key needed by a Coordinator to join the cluster.
func (c *Core) GetK8SCertificateKey() (string, error) {
return c.kube.GetKubeadmCertificateKey()
}
// InitCluster initializes the cluster, stores the join args, and returns the kubeconfig.
func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountURI string) ([]byte, error) {
var nodeName string
@ -123,7 +128,7 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountUR
}
// JoinCluster lets a Node join the cluster.
func (c *Core) JoinCluster(args kubeadm.BootstrapTokenDiscovery) error {
func (c *Core) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, certKey string, peerRole role.Role) error {
c.zaplogger.Info("Joining kubernetes cluster")
nodeVPNIP, err := c.vpn.GetInterfaceIP()
if err != nil {
@ -155,14 +160,16 @@ func (c *Core) JoinCluster(args kubeadm.BootstrapTokenDiscovery) error {
}
}
if err := c.kube.JoinCluster(&args, k8sCompliantHostname(nodeName), nodeIP, providerID); err != nil {
c.zaplogger.Info("k8s Join data", zap.String("nodename", nodeName), zap.String("nodeIP", nodeIP), zap.String("nodeVPNIP", nodeVPNIP), zap.String("provid", providerID))
// we need to pass the VPNIP for another control-plane, otherwise etcd will bind itself to the wrong IP address and fails
if err := c.kube.JoinCluster(args, k8sCompliantHostname(nodeName), nodeIP, nodeVPNIP, providerID, certKey, peerRole); err != nil {
c.zaplogger.Error("Joining kubernetes cluster failed", zap.Error(err))
return err
}
c.zaplogger.Info("Joined kubernetes cluster")
// set role in cloud provider metadata for autoconfiguration
if c.metadata.Supported() {
if err := c.metadata.SignalRole(context.TODO(), role.Node); err != nil {
if err := c.metadata.SignalRole(context.TODO(), peerRole); err != nil {
c.zaplogger.Info("unable to update role in cloud provider metadata", zap.Error(err))
}
}
@ -175,9 +182,11 @@ type Cluster interface {
// InitCluster bootstraps a new cluster with the current node being the master, returning the arguments required to join the cluster.
InitCluster(kubernetes.InitClusterInput) (*kubeadm.BootstrapTokenDiscovery, error)
// JoinCluster will join the current node to an existing cluster.
JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, providerID string) error
JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, peerRole role.Role) error
// GetKubeconfig reads the kubeconfig from the filesystem. Only succeeds after cluster is initialized.
GetKubeconfig() ([]byte, error)
// GetKubeadmCertificateKey returns the 64-byte hex string key needed to join the cluster as control-plane. This function must be executed on a control-plane.
GetKubeadmCertificateKey() (string, error)
}
// ClusterFake behaves like a real cluster, but does not actually initialize or join kubernetes.
@ -193,7 +202,7 @@ func (c *ClusterFake) InitCluster(kubernetes.InitClusterInput) (*kubeadm.Bootstr
}
// JoinCluster will fake joining the current node to an existing cluster.
func (c *ClusterFake) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, providerID string) error {
func (c *ClusterFake) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, _ role.Role) error {
return nil
}
@ -202,6 +211,11 @@ func (c *ClusterFake) GetKubeconfig() ([]byte, error) {
return []byte("kubeconfig"), nil
}
// GetKubeadmCertificateKey fakes generating a certificateKey.
func (c *ClusterFake) GetKubeadmCertificateKey() (string, error) {
return "controlPlaneCertficateKey", nil
}
// k8sCompliantHostname transforms a hostname to an RFC 1123 compliant, lowercase subdomain as required by kubernetes node names.
// The following regex is used by k8s for validation: /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/ .
// Only a simple heuristic is used for now (to lowercase, replace underscores).

11
coordinator/core/cluster_test.go
View File

@ -9,6 +9,7 @@ import (
"github.com/edgelesssys/constellation/coordinator/attestation/simulator"
"github.com/edgelesssys/constellation/coordinator/kubernetes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -287,12 +288,12 @@ func TestJoinCluster(t *testing.T) {
core, err := NewCore(&tc.vpn, &tc.cluster, &tc.metadata, &tc.cloudControllerManager, &tc.cloudNodeManager, &tc.clusterAutoscaler, nil, zapLogger, simulator.OpenSimulatedTPM, nil, file.NewHandler(afero.NewMemMapFs()))
require.NoError(err)
joinReq := kubeadm.BootstrapTokenDiscovery{
joinReq := &kubeadm.BootstrapTokenDiscovery{
APIServerEndpoint: "192.0.2.0:6443",
Token: "someToken",
CACertHashes: []string{"someHash"},
}
err = core.JoinCluster(joinReq)
err = core.JoinCluster(joinReq, "", role.Node)
if tc.expectErr {
assert.Error(err)
@ -354,7 +355,7 @@ func (c *clusterStub) InitCluster(in kubernetes.InitClusterInput) (*kubeadm.Boot
return &c.initJoinArgs, c.initErr
}
func (c *clusterStub) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, providerID string) error {
func (c *clusterStub) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, nodeVPNIP, providerID, certKey string, _ role.Role) error {
c.joinClusterArgs = append(c.joinClusterArgs, joinClusterArgs{
args: args,
nodeName: nodeName,
@ -369,6 +370,10 @@ func (c *clusterStub) GetKubeconfig() ([]byte, error) {
return c.kubeconfig, c.getKubeconfigErr
}
func (c *clusterStub) GetKubeadmCertificateKey() (string, error) {
return "dummy", nil
}
type prepareInstanceRequest struct {
instance Instance
vpnIP string

1
coordinator/kubernetes/k8sapi/joinargs.go
View File

@ -32,6 +32,7 @@ func ParseJoinCommand(joinCommand string) (*kubeadm.BootstrapTokenDiscovery, err
flags.StringVar(&result.Token, "token", "", "")
flags.StringVar(&caCertHash, "discovery-token-ca-cert-hash", "", "")
flags.Bool("control-plane", false, "")
flags.String("certificate-key", "", "")
if err := flags.Parse(argv[3:]); err != nil {
return nil, fmt.Errorf("parsing flag arguments failed: %v %w", argv, err)
}

10
coordinator/kubernetes/k8sapi/kubeadm_config.go
View File

@ -179,6 +179,16 @@ func (k *KubeadmJoinYAML) SetProviderID(providerID string) {
k.KubeletConfiguration.ProviderID = providerID
}
func (k *KubeadmJoinYAML) SetControlPlane(advertiseAddress string, certificateKey string) {
k.JoinConfiguration.ControlPlane = &kubeadm.JoinControlPlane{
LocalAPIEndpoint: kubeadm.APIEndpoint{
AdvertiseAddress: advertiseAddress,
BindPort: 6443,
},
CertificateKey: certificateKey,
}
}
func (k *KubeadmJoinYAML) Marshal() ([]byte, error) {
return resources.MarshalK8SResources(k)
}

1
coordinator/kubernetes/k8sapi/kubeadm_config_test.go
View File

@ -103,6 +103,7 @@ func TestJoinConfiguration(t *testing.T) {
c.SetToken("token")
c.AppendDiscoveryTokenCaCertHash("discovery-token-ca-cert-hash")
c.SetProviderID("somecloudprovider://instance-id")
c.SetControlPlane("192.0.2.0", "11111111111111111111111111111111111")
return c
}(),
},

28
coordinator/kubernetes/k8sapi/util.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"os"
"os/exec"
"regexp"
"strings"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
@ -29,6 +30,7 @@ type ClusterUtil interface {
SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error
SetupCloudNodeManager(kubectl Client, cloudNodeManagerConfiguration resources.Marshaler) error
RestartKubelet() error
GetControlPlaneJoinCertificateKey() (string, error)
}
type KubernetesUtil struct{}
@ -167,3 +169,29 @@ func (k *KubernetesUtil) JoinCluster(joinConfig []byte) error {
func (k *KubernetesUtil) RestartKubelet() error {
return RestartSystemdUnit("kubelet.service")
}
// GetControlPlaneJoinCertificateKey return the key which can be used in combination with the joinArgs
// to join the Cluster as control-plane.
func (k *KubernetesUtil) GetControlPlaneJoinCertificateKey() (string, error) {
// Key will be valid for 1h (no option to reduce the duration).
// https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-init-phase/#cmd-phase-upload-certs
output, err := exec.Command("kubeadm", "init", "phase", "upload-certs", "--upload-certs").Output()
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return "", fmt.Errorf("kubeadm upload-certs failed (code %v) with: %s", exitErr.ExitCode(), exitErr.Stderr)
}
return "", fmt.Errorf("kubeadm upload-certs failed: %w", err)
}
// Example output:
/*
[upload-certs] Storing the certificates in ConfigMap "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
9555b74008f24687eb964bd90a164ecb5760a89481d9c55a77c129b7db438168
*/
key := regexp.MustCompile("[a-f0-9]{64}").FindString(string(output))
if key == "" {
return "", fmt.Errorf("failed to parse kubeadm output: %s", string(output))
}
return key, nil
}

13
coordinator/kubernetes/kubernetes.go
View File

@ -6,6 +6,7 @@ import (
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/spf13/afero"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
@ -103,14 +104,17 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
}
// JoinCluster joins existing kubernetes cluster.
func (k *KubeWrapper) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeIP, providerID string) error {
func (k *KubeWrapper) JoinCluster(args *kubeadm.BootstrapTokenDiscovery, nodeName, nodeInternalIP, nodeVPNIP, providerID, certKey string, peerRole role.Role) error {
joinConfig := k.configProvider.JoinConfiguration()
joinConfig.SetApiServerEndpoint(args.APIServerEndpoint)
joinConfig.SetToken(args.Token)
joinConfig.AppendDiscoveryTokenCaCertHash(args.CACertHashes[0])
joinConfig.SetNodeIP(nodeIP)
joinConfig.SetNodeIP(nodeInternalIP)
joinConfig.SetNodeName(nodeName)
joinConfig.SetProviderID(providerID)
if peerRole == role.Coordinator {
joinConfig.SetControlPlane(nodeVPNIP, certKey)
}
joinConfigYAML, err := joinConfig.Marshal()
if err != nil {
return fmt.Errorf("encoding kubeadm join configuration as YAML failed: %w", err)
@ -135,6 +139,11 @@ func (k *KubeWrapper) GetKubeconfig() ([]byte, error) {
return []byte(strings.ReplaceAll(string(kubeconf), "127.0.0.1:16443", "10.118.0.1:6443")), nil
}
// GetKubeadmCertificateKey return the key needed to join the Cluster as Control-Plane (has to be executed on a control-plane; errors otherwise).
func (k *KubeWrapper) GetKubeadmCertificateKey() (string, error) {
return k.clusterUtil.GetControlPlaneJoinCertificateKey()
}
type fakeK8SClient struct {
kubeconfig []byte
}

7
coordinator/kubernetes/kubernetes_test.go
View File

@ -6,6 +6,7 @@ import (
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/coordinator/role"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
@ -64,6 +65,10 @@ func (s *stubClusterUtil) RestartKubelet() error {
return s.restartKubeletErr
}
func (s *stubClusterUtil) GetControlPlaneJoinCertificateKey() (string, error) {
return "", nil
}
type stubConfigProvider struct {
InitConfig k8sapi.KubeadmInitYAML
JoinConfig k8sapi.KubeadmJoinYAML
@ -236,7 +241,7 @@ func TestJoinCluster(t *testing.T) {
require := require.New(t)
kube := New(&tc.clusterUtil, &stubConfigProvider{}, &client)
err := kube.JoinCluster(joinCommand, instanceName, nodeVPNIP, coordinatorProviderID)
err := kube.JoinCluster(joinCommand, instanceName, nodeVPNIP, nodeVPNIP, coordinatorProviderID, "", role.Node)
if tc.expectErr {
assert.Error(err)
return