Distribute k8s CA certificates and key over join-service

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

joinservice/internal/kubeadm/keymanager.go

@@ -1,75 +0,0 @@
-package kubeadm
-
-import (
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/edgelesssys/constellation/internal/constants"
-	"github.com/edgelesssys/constellation/internal/logger"
-	clientset "k8s.io/client-go/kubernetes"
-	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
-	"k8s.io/kubernetes/cmd/kubeadm/app/phases/copycerts"
-	"k8s.io/utils/clock"
-)
-
-// certificateKeyTTL is the time a certificate key is valid for.
-const certificateKeyTTL = time.Hour
-
-// keyManager handles creation of certificate encryption keys.
-type keyManager struct {
-	mux            sync.Mutex
-	key            string
-	expirationDate time.Time
-	clock          clock.Clock
-	client         clientset.Interface
-	log            *logger.Logger
-}
-
-func newKeyManager(client clientset.Interface, log *logger.Logger) *keyManager {
-	return &keyManager{
-		clock:  clock.RealClock{},
-		client: client,
-		log:    log,
-	}
-}
-
-// getCertificatetKey returns the encryption key to use for uploading PKI certificates to Kubernetes.
-// A Key is cached for one hour, but its expiration date is extended by two minutes if a request is made
-// within two minutes of the key expiring to avoid just-expired keys.
-// This is necessary since uploading a certificate with a different key overwrites any others.
-// This means we can no longer decrypt the certificates using an old key.
-func (k *keyManager) getCertificatetKey() (string, error) {
-	k.mux.Lock()
-	defer k.mux.Unlock()
-
-	switch {
-	case k.key == "" || k.expirationDate.Before(k.clock.Now()):
-		// key was not yet generated, or has expired
-		// generate a new key and set TTL
-		key, err := copycerts.CreateCertificateKey()
-		if err != nil {
-			return "", fmt.Errorf("couldn't create control plane certificate key: %w", err)
-		}
-		k.expirationDate = k.clock.Now().Add(certificateKeyTTL)
-		k.key = key
-		k.log.Infof("Uploading certs to Kubernetes")
-		cfg := &kubeadmapi.InitConfiguration{
-			ClusterConfiguration: kubeadmapi.ClusterConfiguration{
-				CertificatesDir: constants.KubeadmCertificateDir,
-			},
-		}
-		if err := copycerts.UploadCerts(k.client, cfg, key); err != nil {
-			return "", fmt.Errorf("uploading certs: %w", err)
-		}
-	case k.expirationDate.After(k.clock.Now()):
-		// key is still valid
-		// if TTL is less than 2 minutes away, increase it by 2 minutes
-		// this is to avoid the key expiring too soon when a node uses it to join the cluster
-		if k.expirationDate.Sub(k.clock.Now()) < 2*time.Minute {
-			k.expirationDate = k.expirationDate.Add(2 * time.Minute)
-		}
-	}
-
-	return k.key, nil
-}

joinservice/internal/kubeadm/keymanager_test.go

@@ -1,96 +0,0 @@
-package kubeadm
-
-import (
-	"testing"
-	"time"
-
-	"github.com/edgelesssys/constellation/internal/logger"
-	"github.com/stretchr/testify/assert"
-	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/kubernetes/fake"
-	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
-	fakecorev1 "k8s.io/client-go/kubernetes/typed/core/v1/fake"
-	"k8s.io/utils/clock"
-	testclock "k8s.io/utils/clock/testing"
-)
-
-func TestKeyManager(t *testing.T) {
-	testCases := map[string]struct {
-		clock       clock.Clock
-		client      clientset.Interface
-		ttl         time.Time
-		key         string
-		shouldReuse bool
-		wantErr     bool
-	}{
-		"no key exists": {
-			clock:  testclock.NewFakeClock(time.Time{}),
-			client: fake.NewSimpleClientset(),
-		},
-		"key exists and is valid": {
-			clock:       testclock.NewFakeClock(time.Time{}),
-			client:      fake.NewSimpleClientset(),
-			ttl:         time.Time{}.Add(time.Hour),
-			key:         "key",
-			shouldReuse: true,
-		},
-		"key has expired": {
-			clock:  testclock.NewFakeClock(time.Time{}.Add(time.Hour)),
-			client: fake.NewSimpleClientset(),
-			ttl:    time.Time{},
-			key:    "key",
-		},
-		"key expires in the next 30 seconds": {
-			clock:       testclock.NewFakeClock(time.Time{}),
-			client:      fake.NewSimpleClientset(),
-			ttl:         time.Time{}.Add(30 * time.Second),
-			key:         "key",
-			shouldReuse: true,
-		},
-		"uploading certs fails": {
-			clock: testclock.NewFakeClock(time.Time{}),
-			client: &failingClient{
-				fake.NewSimpleClientset(),
-			},
-		},
-	}
-
-	for name, tc := range testCases {
-		t.Run(name, func(t *testing.T) {
-			assert := assert.New(t)
-
-			km := &keyManager{
-				expirationDate: tc.ttl,
-				key:            tc.key,
-				clock:          tc.clock,
-				log:            logger.NewTest(t),
-				client:         fake.NewSimpleClientset(),
-			}
-
-			key, err := km.getCertificatetKey()
-			if tc.wantErr {
-				assert.Error(err)
-				return
-			}
-			assert.NoError(err)
-			assert.True(km.expirationDate.After(tc.clock.Now().Add(2 * time.Minute)))
-
-			if tc.shouldReuse {
-				assert.Equal(tc.key, key)
-			} else {
-				assert.Equal(km.key, key)
-				assert.NotEqual(tc.key, key)
-			}
-		})
-	}
-}
-
-type failingClient struct {
-	*fake.Clientset
-}
-
-func (f *failingClient) CoreV1() corev1.CoreV1Interface {
-	return &failingCoreV1{
-		&fakecorev1.FakeCoreV1{Fake: &f.Clientset.Fake},
-	}
-}

joinservice/internal/kubeadm/kubeadm.go

@@ -3,6 +3,7 @@ package kubeadm
 import (
 	"errors"
 	"fmt"
+	"path/filepath"
 	"time"
 
 	"github.com/edgelesssys/constellation/internal/constants"
@@ -17,6 +18,7 @@ import (
 	bootstraputil "k8s.io/cluster-bootstrap/token/util"
 	bootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/apis/bootstraptoken/v1"
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
+	kubeconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin"
@@ -26,7 +28,6 @@ import (
 type Kubeadm struct {
 	apiServerEndpoint string
 	log               *logger.Logger
-	keyManager        *keyManager
 	client            clientset.Interface
 	file              file.Handler
 }
@@ -46,7 +47,6 @@ func New(apiServerEndpoint string, log *logger.Logger) (*Kubeadm, error) {
 	return &Kubeadm{
 		apiServerEndpoint: apiServerEndpoint,
 		log:               log,
-		keyManager:        newKeyManager(client, log),
 		client:            client,
 		file:              file,
 	}, nil
@@ -108,13 +108,39 @@ func (k *Kubeadm) GetJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscov
 	}, nil
 }
 
-// GetControlPlaneCertificateKey uploads Kubernetes encrypted CA certificates to Kubernetes and returns the decryption key.
-// The key can be used by new nodes to join the cluster as a control plane node.
-func (k *Kubeadm) GetControlPlaneCertificateKey() (string, error) {
-	k.log.Infof("Creating new random control plane certificate key (or returning cached key)")
-	key, err := k.keyManager.getCertificatetKey()
-	if err != nil {
-		return "", fmt.Errorf("couldn't create control plane certificate key: %w", err)
+// GetControlPlaneCertificatesAndKeys loads the Kubernetes CA certificates and keys.
+func (k *Kubeadm) GetControlPlaneCertificatesAndKeys() (map[string][]byte, error) {
+	k.log.Infof("Loading control plane certificates and keys")
+	controlPlaneFiles := make(map[string][]byte)
+
+	keyFilenames := []string{
+		kubeconstants.CAKeyName,
+		kubeconstants.ServiceAccountPrivateKeyName,
+		kubeconstants.FrontProxyCAKeyName,
+		kubeconstants.EtcdCAKeyName,
 	}
-	return key, nil
+	certFilenames := []string{
+		kubeconstants.CACertName,
+		kubeconstants.ServiceAccountPublicKeyName,
+		kubeconstants.FrontProxyCACertName,
+		kubeconstants.EtcdCACertName,
+	}
+
+	for _, keyFilename := range keyFilenames {
+		key, err := k.file.Read(filepath.Join(kubeconstants.KubernetesDir, kubeconstants.DefaultCertificateDir, keyFilename))
+		if err != nil {
+			return nil, err
+		}
+		controlPlaneFiles[keyFilename] = key
+	}
+
+	for _, certFilename := range certFilenames {
+		cert, err := k.file.Read(filepath.Join(kubeconstants.KubernetesDir, kubeconstants.DefaultCertificateDir, certFilename))
+		if err != nil {
+			return nil, err
+		}
+		controlPlaneFiles[certFilename] = cert
+	}
+
+	return controlPlaneFiles, nil
 }

joinservice/internal/kubeadm/kubeadm_test.go

@@ -1,7 +1,7 @@
 package kubeadm
 
 import (
-	"context"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -12,12 +12,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/fake"
-	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
-	fakecorev1 "k8s.io/client-go/kubernetes/typed/core/v1/fake"
-	testclock "k8s.io/utils/clock/testing"
+	kubeconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
 func TestMain(m *testing.M) {
@@ -83,10 +79,9 @@ kind: Config`,
 			require := require.New(t)
 
 			client := &Kubeadm{
-				log:        logger.NewTest(t),
-				keyManager: &keyManager{clock: testclock.NewFakeClock(time.Time{})},
-				file:       file.NewHandler(afero.NewMemMapFs()),
-				client:     fake.NewSimpleClientset(),
+				log:    logger.NewTest(t),
+				file:   file.NewHandler(afero.NewMemMapFs()),
+				client: fake.NewSimpleClientset(),
 			}
 			if tc.adminConf != "" {
 				require.NoError(client.file.Write(constants.CoreOSAdminConfFilename, []byte(tc.adminConf), file.OptNone))
@@ -103,21 +98,70 @@ kind: Config`,
 	}
 }
 
-type failingCoreV1 struct {
-	*fakecorev1.FakeCoreV1
-}
+func TestGetControlPlaneCertificatesAndKeys(t *testing.T) {
+	someData := []byte{0x1, 0x2, 0x3}
 
-func (f *failingCoreV1) Secrets(namespace string) corev1.SecretInterface {
-	return &failingSecretInterface{
-		&fakecorev1.FakeSecrets{Fake: f.FakeCoreV1},
+	testCases := map[string]struct {
+		preExistingFiles map[string][]byte
+		wantErr          bool
+	}{
+		"success": {
+			preExistingFiles: map[string][]byte{
+				kubeconstants.CAKeyName:                    someData,
+				kubeconstants.ServiceAccountPrivateKeyName: someData,
+				kubeconstants.FrontProxyCAKeyName:          someData,
+				kubeconstants.EtcdCAKeyName:                someData,
+				kubeconstants.CACertName:                   someData,
+				kubeconstants.ServiceAccountPublicKeyName:  someData,
+				kubeconstants.FrontProxyCACertName:         someData,
+				kubeconstants.EtcdCACertName:               someData,
+			},
+		},
+		"missing key": {
+			preExistingFiles: map[string][]byte{
+				kubeconstants.CACertName:                  someData,
+				kubeconstants.ServiceAccountPublicKeyName: someData,
+				kubeconstants.FrontProxyCACertName:        someData,
+				kubeconstants.EtcdCACertName:              someData,
+			},
+			wantErr: true,
+		},
+		"missing cert": {
+			preExistingFiles: map[string][]byte{
+				kubeconstants.CAKeyName:                    someData,
+				kubeconstants.ServiceAccountPrivateKeyName: someData,
+				kubeconstants.FrontProxyCAKeyName:          someData,
+				kubeconstants.EtcdCAKeyName:                someData,
+			},
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			client := &Kubeadm{
+				log:    logger.NewTest(t),
+				file:   file.NewHandler(afero.NewMemMapFs()),
+				client: fake.NewSimpleClientset(),
+			}
+			for filename, content := range tc.preExistingFiles {
+				require.NoError(client.file.Write(
+					filepath.Join(kubeconstants.KubernetesDir, kubeconstants.DefaultCertificateDir, filename),
+					content,
+					file.OptNone,
+				))
+			}
+
+			files, err := client.GetControlPlaneCertificatesAndKeys()
+			if tc.wantErr {
+				assert.Error(err)
+			} else {
+				assert.NoError(err)
+				assert.NotNil(files)
+			}
+		})
 	}
 }
-
-type failingSecretInterface struct {
-	*fakecorev1.FakeSecrets
-}
-
-// copycerts.UploadCerts will fail if a secret already exists.
-func (f *failingSecretInterface) Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Secret, error) {
-	return &v1.Secret{}, nil
-}