Create internal package for joinservice

This commit is contained in:
katexochen 2022-07-05 13:42:07 +02:00 committed by Paul Meyer
parent 43eb94b6dc
commit 2083d37b11
10 changed files with 25 additions and 33 deletions

View file

@ -0,0 +1,67 @@
package kms
import (
"context"
"fmt"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/kms/kmsproto"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// Client interacts with Constellation's key management service.
type Client struct {
log *logger.Logger
endpoint string
grpc grpcClient
}
// New creates a new KMS.
func New(log *logger.Logger, endpoint string) Client {
return Client{
log: log,
endpoint: endpoint,
grpc: client{},
}
}
// GetDEK returns a data encryption key for the given UUID.
func (c Client) GetDataKey(ctx context.Context, uuid string, length int) ([]byte, error) {
log := c.log.With(zap.String("diskUUID", uuid), zap.String("endpoint", c.endpoint))
// TODO: update credentials if we enable aTLS on the KMS
// For now this is fine since traffic is only routed through the Constellation cluster
log.Infof("Connecting to KMS at %s", c.endpoint)
conn, err := grpc.DialContext(ctx, c.endpoint, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return nil, err
}
defer conn.Close()
log.Infof("Requesting data key")
res, err := c.grpc.GetDataKey(
ctx,
&kmsproto.GetDataKeyRequest{
DataKeyId: uuid,
Length: uint32(length),
},
conn,
)
if err != nil {
return nil, fmt.Errorf("fetching data encryption key from Constellation KMS: %w", err)
}
log.Infof("Data key request successful")
return res.DataKey, nil
}
type grpcClient interface {
GetDataKey(context.Context, *kmsproto.GetDataKeyRequest, *grpc.ClientConn) (*kmsproto.GetDataKeyResponse, error)
}
type client struct{}
func (c client) GetDataKey(ctx context.Context, req *kmsproto.GetDataKeyRequest, conn *grpc.ClientConn) (*kmsproto.GetDataKeyResponse, error) {
return kmsproto.NewAPIClient(conn).GetDataKey(ctx, req)
}

View file

@ -0,0 +1,66 @@
package kms
import (
"context"
"errors"
"testing"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/kms/kmsproto"
"github.com/stretchr/testify/assert"
"go.uber.org/goleak"
"google.golang.org/grpc"
"google.golang.org/grpc/test/bufconn"
)
type stubClient struct {
getDataKeyErr error
dataKey []byte
}
func (c *stubClient) GetDataKey(context.Context, *kmsproto.GetDataKeyRequest, *grpc.ClientConn) (*kmsproto.GetDataKeyResponse, error) {
return &kmsproto.GetDataKeyResponse{DataKey: c.dataKey}, c.getDataKeyErr
}
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
func TestGetDataKey(t *testing.T) {
testCases := map[string]struct {
client *stubClient
wantErr bool
}{
"GetDataKey success": {
client: &stubClient{dataKey: []byte{0x1, 0x2, 0x3}},
},
"GetDataKey error": {
client: &stubClient{getDataKeyErr: errors.New("error")},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
listener := bufconn.Listen(1)
defer listener.Close()
client := New(
logger.NewTest(t),
listener.Addr().String(),
)
client.grpc = tc.client
res, err := client.GetDataKey(context.Background(), "disk-uuid", 32)
if tc.wantErr {
assert.Error(err)
} else {
assert.NoError(err)
assert.Equal(tc.client.dataKey, res)
}
})
}
}

View file

@ -0,0 +1,127 @@
package kubeadm
import (
"errors"
"fmt"
"time"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/spf13/afero"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
certutil "k8s.io/client-go/util/cert"
bootstraputil "k8s.io/cluster-bootstrap/token/util"
bootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/apis/bootstraptoken/v1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/copycerts"
"k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
"k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin"
)
// Kubeadm manages joining of new nodes.
type Kubeadm struct {
log *logger.Logger
client clientset.Interface
file file.Handler
}
// New creates a new Kubeadm instance.
func New(log *logger.Logger) (*Kubeadm, error) {
config, err := rest.InClusterConfig()
if err != nil {
return nil, fmt.Errorf("failed to get in-cluster config: %w", err)
}
client, err := clientset.NewForConfig(config)
if err != nil {
return nil, fmt.Errorf("failed to create client: %w", err)
}
file := file.NewHandler(afero.NewOsFs())
return &Kubeadm{
log: log,
client: client,
file: file,
}, nil
}
// GetJoinToken creates a new bootstrap (join) token, which a node can use to join the cluster.
func (k *Kubeadm) GetJoinToken(ttl time.Duration) (*kubeadm.BootstrapTokenDiscovery, error) {
k.log.Infof("Generating new random bootstrap token")
rawToken, err := bootstraputil.GenerateBootstrapToken()
if err != nil {
return nil, fmt.Errorf("couldn't generate random token: %w", err)
}
tokenStr, err := bootstraptoken.NewBootstrapTokenString(rawToken)
if err != nil {
return nil, fmt.Errorf("invalid token: %w", err)
}
token := bootstraptoken.BootstrapToken{
Token: tokenStr,
Description: "Bootstrap token generated by Constellation's Activation service",
TTL: &metav1.Duration{Duration: ttl},
}
// create the token in Kubernetes
k.log.Infof("Creating bootstrap token in Kubernetes")
if err := tokenphase.CreateNewTokens(k.client, []bootstraptoken.BootstrapToken{token}); err != nil {
return nil, fmt.Errorf("creating bootstrap token: %w", err)
}
// parse Kubernetes CA certs
k.log.Infof("Preparing join token for new node")
rawConfig, err := k.file.Read(constants.CoreOSAdminConfFilename)
if err != nil {
return nil, fmt.Errorf("loading kubeconfig file: %w", err)
}
config, err := clientcmd.Load(rawConfig)
if err != nil {
return nil, fmt.Errorf("loading kubeconfig file: %w", err)
}
clusterConfig := kubeconfig.GetClusterFromKubeConfig(config)
if clusterConfig == nil {
return nil, errors.New("couldn't get cluster config from kubeconfig file")
}
caCerts, err := certutil.ParseCertsPEM(clusterConfig.CertificateAuthorityData)
if err != nil {
return nil, fmt.Errorf("parsing CA certs: %w", err)
}
publicKeyPins := make([]string, 0, len(caCerts))
for _, caCert := range caCerts {
publicKeyPins = append(publicKeyPins, pubkeypin.Hash(caCert))
}
k.log.Infof("Join token creation successful")
return &kubeadm.BootstrapTokenDiscovery{
Token: tokenStr.String(),
APIServerEndpoint: "10.118.0.1:6443", // This is not HA and should be replaced with the IP of the node issuing the token
CACertHashes: publicKeyPins,
}, nil
}
// GetControlPlaneCertificateKey uploads Kubernetes encrypted CA certificates to Kubernetes and returns the decryption key.
// The key can be used by new nodes to join the cluster as a control plane node.
func (k *Kubeadm) GetControlPlaneCertificateKey() (string, error) {
k.log.Infof("Creating new random control plane certificate key")
key, err := copycerts.CreateCertificateKey()
if err != nil {
return "", fmt.Errorf("couldn't create control plane certificate key: %w", err)
}
k.log.Infof("Uploading certs to Kubernetes")
cfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
CertificatesDir: constants.KubeadmCertificateDir,
},
}
if err := copycerts.UploadCerts(k.client, cfg, key); err != nil {
return "", fmt.Errorf("uploading certs: %w", err)
}
return key, nil
}

View file

@ -0,0 +1,168 @@
package kubeadm
import (
"context"
"testing"
"time"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
fakecorev1 "k8s.io/client-go/kubernetes/typed/core/v1/fake"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
func TestGetJoinToken(t *testing.T) {
validConf := `apiVersion: v1
kind: Config
clusters:
- cluster:
certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvakNDQWVhZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeU1EVXpNREE0TWpJd01Gb1hEVE15TURVeU56QTRNakl3TUZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTmV5CnNubVJQbDYxaXZGWWRIUjFJUjdyRS9PNjNSOVhpVERwM1V4T2tMQzdMaW94bFA0SmRINzdHMUJ4Y2NCSjVISDIKZHBUTklzcjNxMEZ3ckdtK1JVYzdoRjBmZjgwdUtyUVVMN3UrYWlIRU5HSExVSFVnc3V4Tmd1bUxRdnlrRTUzNQp4dWRVSWpVV0g5M3NuRU5GempuWkRZM09SWVdNQ253OVlxMk5CZDdBRktKY1o3WDc3U1I3eStNK3czdGkvQlZpCmNtR1BvRW1WTTV3V0VReFQwYlpxNjcxTXltcmhEenFwbEZ2dkpranFIdVp6dUFhZ0pXWW9nejNsYjZLbCtmdmgKTjBjbFBDMjJyUUJJY01JWDVHdG40bzJ5U2JvQnBoRWNEWkx6TjIyU0tZZ2ViSGQwOU9lcktWdGw5bDl6cmQvVApBWm5jOTNQVCtvWTFsSmdldUE4Q0F3RUFBYU5aTUZjd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZOVmNPNUZZY2NUTVN1SHpJWFZMYlppUnZRVVZNQlVHQTFVZEVRUU8KTUF5Q0NtdDFZbVZ5Ym1WMFpYTXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBTDBsRERnbEsvY1JCNHVoQXJBRwpRSDhOeGtCSnhGNXYrVWMyZVFGa3dRTlB5SkU3QTRMV2p1eEVLN25meWVrTk91c2N2Wm1DQzVJNFhVZHAzb0ptCnZzSVlsN2YvMEFaMUt3d1RvQSt3cFF2QVB1NHlhM251MkZkMC9DVkViazNUZTV1MzRmQkxvL0YzK0Q2dFZLb2gKbVpGYmdoVjdMZms5SlQ4UzZjbGxyYjZkT3dCdGViUDBMQWZJd0hWaDBZNEsyY0thc3ZtU2xtMktpRXdURlBrbgpTSkNWWnI1aUJ3eGFadk1mYlpEaDk1bGZCbEtCVkdMNm5CcWs2TEpKM0VVd0tocTFGZEoyT0lSTkF0em14Z0R3CnNkOWd0SE4rK0pUcnhDa0ZBUTdwVWptdXBjZmpDOWhRRk1HOTRzTzk5elhZd2svTEdhV3FlS0pBYlRiNVdoRWcKYU5ZPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
server: https://127.0.0.1:16443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: kubernetes-admin
name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes`
missingCA := `apiVersion: v1
kind: Config
clusters:
- cluster:
server: https://127.0.0.1:16443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: kubernetes-admin
name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes`
testCases := map[string]struct {
adminConf string
wantErr bool
}{
"success": {
adminConf: validConf,
},
"no certificate-authority-data": {
adminConf: missingCA,
wantErr: true,
},
"no cluster config": {
adminConf: `apiVersion: v1
kind: Config`,
wantErr: true,
},
"invalid config": {
adminConf: "not a config",
wantErr: true,
},
"config does not exist": {
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := &Kubeadm{
log: logger.NewTest(t),
file: file.NewHandler(afero.NewMemMapFs()),
client: fake.NewSimpleClientset(),
}
if tc.adminConf != "" {
require.NoError(client.file.Write(constants.CoreOSAdminConfFilename, []byte(tc.adminConf), file.OptNone))
}
res, err := client.GetJoinToken(time.Minute)
if tc.wantErr {
assert.Error(err)
} else {
assert.NoError(err)
assert.NotNil(res)
}
})
}
}
func TestGetControlPlaneCertificateKey(t *testing.T) {
testCases := map[string]struct {
wantErr bool
client clientset.Interface
}{
"success": {
client: fake.NewSimpleClientset(),
wantErr: false,
},
"failure": {
client: &failingClient{
fake.NewSimpleClientset(),
},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
client := &Kubeadm{
log: logger.NewTest(t),
client: tc.client,
}
_, err := client.GetControlPlaneCertificateKey()
if tc.wantErr {
assert.Error(err)
} else {
assert.NoError(err)
}
})
}
}
type failingClient struct {
*fake.Clientset
}
func (f *failingClient) CoreV1() corev1.CoreV1Interface {
return &failingCoreV1{
&fakecorev1.FakeCoreV1{Fake: &f.Clientset.Fake},
}
}
type failingCoreV1 struct {
*fakecorev1.FakeCoreV1
}
func (f *failingCoreV1) Secrets(namespace string) corev1.SecretInterface {
return &failingSecretInterface{
&fakecorev1.FakeSecrets{Fake: f.FakeCoreV1},
}
}
type failingSecretInterface struct {
*fakecorev1.FakeSecrets
}
// copycerts.UploadCerts will fail if a secret already exists.
func (f *failingSecretInterface) Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Secret, error) {
return &v1.Secret{}, nil
}

View file

@ -0,0 +1,119 @@
package kubernetesca
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"time"
"github.com/edgelesssys/constellation/bootstrapper/util"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/logger"
)
const (
caCertFilename = "/etc/kubernetes/pki/ca.crt"
caKeyFilename = "/etc/kubernetes/pki/ca.key"
)
// KubernetesCA handles signing of certificates using the Kubernetes root CA.
type KubernetesCA struct {
log *logger.Logger
file file.Handler
}
// New creates a new KubernetesCA.
func New(log *logger.Logger, fileHandler file.Handler) *KubernetesCA {
return &KubernetesCA{
log: log,
file: fileHandler,
}
}
// GetCertificate creates a certificate for a node and signs it using the Kubernetes root CA.
func (c KubernetesCA) GetCertificate(nodeName string) (cert []byte, key []byte, err error) {
c.log.Debugf("Loading Kubernetes CA certificate")
parentCertRaw, err := c.file.Read(caCertFilename)
if err != nil {
return nil, nil, err
}
parentCertPEM, _ := pem.Decode(parentCertRaw)
parentCert, err := x509.ParseCertificate(parentCertPEM.Bytes)
if err != nil {
return nil, nil, err
}
c.log.Debugf("Loading Kubernetes CA private key")
parentKeyRaw, err := c.file.Read(caKeyFilename)
if err != nil {
return nil, nil, err
}
parentKeyPEM, _ := pem.Decode(parentKeyRaw)
var parentKey any
switch parentKeyPEM.Type {
case "EC PRIVATE KEY":
parentKey, err = x509.ParseECPrivateKey(parentKeyPEM.Bytes)
case "RSA PRIVATE KEY":
parentKey, err = x509.ParsePKCS1PrivateKey(parentKeyPEM.Bytes)
case "PRIVATE KEY":
parentKey, err = x509.ParsePKCS8PrivateKey(parentKeyPEM.Bytes)
default:
return nil, nil, fmt.Errorf("unsupported key type %q", parentCertPEM.Type)
}
if err != nil {
return nil, nil, err
}
c.log.Infof("Creating kubelet private key")
privK, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, nil, err
}
keyBytes, err := x509.MarshalECPrivateKey(privK)
if err != nil {
return nil, nil, err
}
kubeletKey := pem.EncodeToMemory(&pem.Block{
Type: "EC PRIVATE KEY",
Bytes: keyBytes,
})
c.log.Infof("Creating kubelet certificate")
serialNumber, err := util.GenerateCertificateSerialNumber()
if err != nil {
return nil, nil, err
}
now := time.Now()
// Create the kubelet certificate
// For a reference on the certificate fields, see: https://kubernetes.io/docs/setup/best-practices/certificates/
certTmpl := &x509.Certificate{
SerialNumber: serialNumber,
NotBefore: now.Add(-2 * time.Hour),
NotAfter: now.Add(24 * 365 * time.Hour),
Subject: pkix.Name{
Organization: []string{"system:nodes"},
CommonName: fmt.Sprintf("system:node:%s", nodeName),
},
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{
x509.ExtKeyUsageClientAuth,
},
IsCA: false,
BasicConstraintsValid: true,
}
certRaw, err := x509.CreateCertificate(rand.Reader, certTmpl, parentCert, &privK.PublicKey, parentKey)
if err != nil {
return nil, nil, err
}
kubeletCert := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: certRaw,
})
return kubeletCert, kubeletKey, nil
}

View file

@ -0,0 +1,206 @@
package kubernetesca
import (
"crypto"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"testing"
"time"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
func TestGetCertificate(t *testing.T) {
ecCert, ecKey := mustCreateCert(mustCreateECKey)
rsaCert, rsaKey := mustCreateCert(mustCreateRSAKey)
testCert, testKey := mustCreateCert(mustCreatePKCS8Key)
unsupportedKey := []byte(`-----BEGIN SOME KEY-----
Q29uc3RlbGxhdGlvbg==
-----END SOME KEY-----`)
invalidKey := []byte(`-----BEGIN PRIVATE KEY-----
Q29uc3RlbGxhdGlvbg==
-----END PRIVATE KEY-----`)
invalidCert := []byte(`-----BEGIN CERTIFICATE-----
Q29uc3RlbGxhdGlvbg==
-----END CERTIFICATE-----`)
testCases := map[string]struct {
caCert []byte
caKey []byte
wantErr bool
}{
"success ec key": {
caCert: ecCert,
caKey: ecKey,
},
"success rsa key": {
caCert: rsaCert,
caKey: rsaKey,
},
"success any key": {
caCert: testCert,
caKey: testKey,
},
"unsupported key": {
caCert: ecCert,
caKey: unsupportedKey,
wantErr: true,
},
"invalid key": {
caCert: ecCert,
caKey: invalidKey,
wantErr: true,
},
"invalid certificate": {
caCert: invalidCert,
caKey: ecKey,
wantErr: true,
},
"no ca certificate": {
caKey: ecKey,
wantErr: true,
},
"no ca key": {
caCert: ecCert,
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
file := file.NewHandler(afero.NewMemMapFs())
if len(tc.caCert) > 0 {
require.NoError(file.Write(caCertFilename, tc.caCert, 0o644))
}
if len(tc.caKey) > 0 {
require.NoError(file.Write(caKeyFilename, tc.caKey, 0o644))
}
ca := New(
logger.NewTest(t),
file,
)
nodeName := "test"
kubeCert, kubeKey, err := ca.GetCertificate(nodeName)
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
certPEM, _ := pem.Decode(kubeCert)
require.NotNil(certPEM)
cert, err := x509.ParseCertificate(certPEM.Bytes)
require.NoError(err)
assert.Equal("system:node:"+nodeName, cert.Subject.CommonName)
assert.Equal("system:nodes", cert.Subject.Organization[0])
assert.Equal(x509.KeyUsageDigitalSignature|x509.KeyUsageKeyEncipherment, cert.KeyUsage)
assert.Equal(x509.ExtKeyUsageClientAuth, cert.ExtKeyUsage[0])
assert.False(cert.IsCA)
assert.True(cert.BasicConstraintsValid)
keyPEM, _ := pem.Decode(kubeKey)
require.NotNil(keyPEM)
key, err := x509.ParseECPrivateKey(keyPEM.Bytes)
require.NoError(err)
require.IsType(&ecdsa.PublicKey{}, cert.PublicKey)
assert.Equal(&key.PublicKey, cert.PublicKey.(*ecdsa.PublicKey))
})
}
}
func mustCreateCert(getKey func() (crypto.PrivateKey, []byte)) ([]byte, []byte) {
caPriv, keyPEM := getKey()
caTemplate := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: "kubernetes",
},
NotBefore: time.Now().Add(-2 * time.Hour),
IsCA: true,
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
}
caCert, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, publicKey(caPriv), caPriv)
if err != nil {
panic(err)
}
caCertPEM := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: caCert,
})
return caCertPEM, keyPEM
}
func mustCreateECKey() (crypto.PrivateKey, []byte) {
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
panic(err)
}
keyBytes, err := x509.MarshalECPrivateKey(key)
if err != nil {
panic(err)
}
return key, pem.EncodeToMemory(&pem.Block{
Type: "EC PRIVATE KEY",
Bytes: keyBytes,
})
}
func mustCreatePKCS8Key() (crypto.PrivateKey, []byte) {
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
panic(err)
}
keyBytes, err := x509.MarshalPKCS8PrivateKey(key)
if err != nil {
panic(err)
}
return key, pem.EncodeToMemory(&pem.Block{
Type: "PRIVATE KEY",
Bytes: keyBytes,
})
}
func mustCreateRSAKey() (crypto.PrivateKey, []byte) {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic(err)
}
keyBytes := x509.MarshalPKCS1PrivateKey(key)
return key, pem.EncodeToMemory(&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: keyBytes,
})
}
func publicKey(priv crypto.PrivateKey) any {
switch k := priv.(type) {
case *rsa.PrivateKey:
return &k.PublicKey
case *ecdsa.PrivateKey:
return &k.PublicKey
default:
return nil
}
}

View file

@ -0,0 +1,148 @@
package server
import (
"context"
"fmt"
"net"
"path/filepath"
"time"
attestationtypes "github.com/edgelesssys/constellation/internal/attestation/types"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/grpc/grpclog"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/joinservice/joinproto"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/status"
kubeadmv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
// Server implements the core logic of Constellation's node activation service.
type Server struct {
log *logger.Logger
file file.Handler
joinTokenGetter joinTokenGetter
dataKeyGetter dataKeyGetter
ca certificateAuthority
joinproto.UnimplementedAPIServer
}
// New initializes a new Server.
func New(log *logger.Logger, fileHandler file.Handler, ca certificateAuthority, joinTokenGetter joinTokenGetter, dataKeyGetter dataKeyGetter) *Server {
return &Server{
log: log,
file: fileHandler,
joinTokenGetter: joinTokenGetter,
dataKeyGetter: dataKeyGetter,
ca: ca,
}
}
// Run starts the gRPC server on the given port, using the provided tlsConfig.
func (s *Server) Run(creds credentials.TransportCredentials, port string) error {
s.log.WithIncreasedLevel(zap.WarnLevel).Named("gRPC").ReplaceGRPCLogger()
grpcServer := grpc.NewServer(
grpc.Creds(creds),
s.log.Named("gRPC").GetServerUnaryInterceptor(),
)
joinproto.RegisterAPIServer(grpcServer, s)
lis, err := net.Listen("tcp", net.JoinHostPort("", port))
if err != nil {
return fmt.Errorf("failed to listen: %s", err)
}
s.log.Infof("Starting activation service on %s", lis.Addr().String())
return grpcServer.Serve(lis)
}
// IssueJoinTicket handles activation requests of Constellation nodes.
// A node will receive:
// - stateful disk encryption key.
// - Kubernetes join token.
// - cluster and owner ID to taint the node as initialized.
// In addition, control plane nodes receive:
// - a decryption key for CA certificates uploaded to the Kubernetes cluster.
func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTicketRequest) (resp *joinproto.IssueJoinTicketResponse, retErr error) {
s.log.Infof("IssueJoinTicket called")
defer func() {
if retErr != nil {
s.log.Errorf("IssueJoinTicket failed: %s", retErr)
retErr = fmt.Errorf("IssueJoinTicket failed: %w", retErr)
}
}()
log := s.log.With(zap.String("peerAddress", grpclog.PeerAddrFromContext(ctx)))
log.Infof("Loading IDs")
var id attestationtypes.ID
if err := s.file.ReadJSON(filepath.Join(constants.ServiceBasePath, constants.IDFilename), &id); err != nil {
log.With(zap.Error(err)).Errorf("Unable to load IDs")
return nil, status.Errorf(codes.Internal, "unable to load IDs: %s", err)
}
log.Infof("Requesting disk encryption key")
stateDiskKey, err := s.dataKeyGetter.GetDataKey(ctx, req.DiskUuid, constants.StateDiskKeyLength)
if err != nil {
log.With(zap.Error(err)).Errorf("Unable to get key for stateful disk")
return nil, status.Errorf(codes.Internal, "unable to get key for stateful disk: %s", err)
}
log.Infof("Creating Kubernetes join token")
kubeArgs, err := s.joinTokenGetter.GetJoinToken(constants.KubernetesJoinTokenTTL)
if err != nil {
log.With(zap.Error(err)).Errorf("Unable to generate Kubernetes join arguments")
return nil, status.Errorf(codes.Internal, "unable to generate Kubernetes join arguments: %s", err)
}
log.Infof("Creating signed kubelet certificate")
kubeletCert, kubeletKey, err := s.ca.GetCertificate(req.NodeName)
if err != nil {
return nil, status.Errorf(codes.Internal, "unable to generate kubelet certificate: %s", err)
}
var certKey string
if req.IsControlPlane {
log.Infof("Creating control plane certificate key")
certKey, err = s.joinTokenGetter.GetControlPlaneCertificateKey()
if err != nil {
return nil, fmt.Errorf("ActivateControlPlane failed: %w", err)
}
}
s.log.Infof("IssueJoinTicket successful")
return &joinproto.IssueJoinTicketResponse{
StateDiskKey: stateDiskKey,
ClusterId: id.Cluster,
OwnerId: id.Owner,
ApiServerEndpoint: kubeArgs.APIServerEndpoint,
Token: kubeArgs.Token,
DiscoveryTokenCaCertHash: kubeArgs.CACertHashes[0],
KubeletCert: kubeletCert,
KubeletKey: kubeletKey,
CertificateKey: certKey,
}, nil
}
// joinTokenGetter returns Kubernetes bootstrap (join) tokens.
type joinTokenGetter interface {
// GetJoinToken returns a bootstrap (join) token.
GetJoinToken(ttl time.Duration) (*kubeadmv1.BootstrapTokenDiscovery, error)
GetControlPlaneCertificateKey() (string, error)
}
// dataKeyGetter interacts with Constellation's key management system to retrieve keys.
type dataKeyGetter interface {
// GetDataKey returns a key derived from Constellation's KMS.
GetDataKey(ctx context.Context, uuid string, length int) ([]byte, error)
}
type certificateAuthority interface {
// GetCertificate returns a certificate and private key, signed by the issuer.
GetCertificate(nodeName string) (kubeletCert []byte, kubeletKey []byte, err error)
}

View file

@ -0,0 +1,194 @@
package server
import (
"context"
"encoding/json"
"errors"
"path/filepath"
"testing"
"time"
attestationtypes "github.com/edgelesssys/constellation/internal/attestation/types"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/logger"
"github.com/edgelesssys/constellation/joinservice/joinproto"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
kubeadmv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
func TestIssueJoinTicket(t *testing.T) {
someErr := errors.New("error")
testKey := []byte{0x1, 0x2, 0x3}
testCert := []byte{0x4, 0x5, 0x6}
testID := attestationtypes.ID{
Owner: []byte{0x4, 0x5, 0x6},
Cluster: []byte{0x7, 0x8, 0x9},
}
testJoinToken := &kubeadmv1.BootstrapTokenDiscovery{
APIServerEndpoint: "192.0.2.1",
CACertHashes: []string{"hash"},
Token: "token",
}
testCases := map[string]struct {
isControlPlane bool
kubeadm stubTokenGetter
kms stubKeyGetter
ca stubCA
id []byte
wantErr bool
}{
"worker node": {
kubeadm: stubTokenGetter{token: testJoinToken},
kms: stubKeyGetter{dataKey: testKey},
ca: stubCA{cert: testCert, key: testKey},
id: mustMarshalID(testID),
},
"GetDataKey fails": {
kubeadm: stubTokenGetter{token: testJoinToken},
kms: stubKeyGetter{getDataKeyErr: someErr},
ca: stubCA{cert: testCert, key: testKey},
id: mustMarshalID(testID),
wantErr: true,
},
"loading IDs fails": {
kubeadm: stubTokenGetter{token: testJoinToken},
kms: stubKeyGetter{dataKey: testKey},
ca: stubCA{cert: testCert, key: testKey},
id: []byte{0x1, 0x2, 0x3},
wantErr: true,
},
"no ID file": {
kubeadm: stubTokenGetter{token: testJoinToken},
kms: stubKeyGetter{dataKey: testKey},
ca: stubCA{cert: testCert, key: testKey},
wantErr: true,
},
"GetJoinToken fails": {
kubeadm: stubTokenGetter{getJoinTokenErr: someErr},
kms: stubKeyGetter{dataKey: testKey},
ca: stubCA{cert: testCert, key: testKey},
id: mustMarshalID(testID),
wantErr: true,
},
"GetCertificate fails": {
kubeadm: stubTokenGetter{token: testJoinToken},
kms: stubKeyGetter{dataKey: testKey},
ca: stubCA{getCertErr: someErr},
id: mustMarshalID(testID),
wantErr: true,
},
"control plane": {
isControlPlane: true,
kubeadm: stubTokenGetter{token: testJoinToken, certificateKey: "test"},
kms: stubKeyGetter{dataKey: testKey},
ca: stubCA{cert: testCert, key: testKey},
id: mustMarshalID(testID),
},
"GetControlPlaneCertificateKey fails": {
isControlPlane: true,
kubeadm: stubTokenGetter{token: testJoinToken, certificateKeyErr: someErr},
kms: stubKeyGetter{dataKey: testKey},
ca: stubCA{cert: testCert, key: testKey},
id: mustMarshalID(testID),
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
file := file.NewHandler(afero.NewMemMapFs())
if len(tc.id) > 0 {
require.NoError(file.Write(filepath.Join(constants.ServiceBasePath, constants.IDFilename), tc.id, 0o644))
}
api := New(
logger.NewTest(t),
file,
tc.ca,
tc.kubeadm,
tc.kms,
)
req := &joinproto.IssueJoinTicketRequest{
DiskUuid: "uuid",
NodeName: "test",
IsControlPlane: tc.isControlPlane,
}
resp, err := api.IssueJoinTicket(context.Background(), req)
if tc.wantErr {
assert.Error(err)
return
}
var expectedIDs attestationtypes.ID
require.NoError(json.Unmarshal(tc.id, &expectedIDs))
require.NoError(err)
assert.Equal(tc.kms.dataKey, resp.StateDiskKey)
assert.Equal(expectedIDs.Cluster, resp.ClusterId)
assert.Equal(expectedIDs.Owner, resp.OwnerId)
assert.Equal(tc.kubeadm.token.APIServerEndpoint, resp.ApiServerEndpoint)
assert.Equal(tc.kubeadm.token.CACertHashes[0], resp.DiscoveryTokenCaCertHash)
assert.Equal(tc.kubeadm.token.Token, resp.Token)
assert.Equal(tc.ca.cert, resp.KubeletCert)
assert.Equal(tc.ca.key, resp.KubeletKey)
if tc.isControlPlane {
assert.Equal(tc.kubeadm.certificateKey, resp.CertificateKey)
}
})
}
}
func mustMarshalID(id attestationtypes.ID) []byte {
b, err := json.Marshal(id)
if err != nil {
panic(err)
}
return b
}
type stubTokenGetter struct {
token *kubeadmv1.BootstrapTokenDiscovery
getJoinTokenErr error
certificateKey string
certificateKeyErr error
}
func (f stubTokenGetter) GetJoinToken(time.Duration) (*kubeadmv1.BootstrapTokenDiscovery, error) {
return f.token, f.getJoinTokenErr
}
func (f stubTokenGetter) GetControlPlaneCertificateKey() (string, error) {
return f.certificateKey, f.certificateKeyErr
}
type stubKeyGetter struct {
dataKey []byte
getDataKeyErr error
}
func (f stubKeyGetter) GetDataKey(context.Context, string, int) ([]byte, error) {
return f.dataKey, f.getDataKeyErr
}
type stubCA struct {
cert []byte
key []byte
getCertErr error
}
func (f stubCA) GetCertificate(string) ([]byte, []byte, error) {
return f.cert, f.key, f.getCertErr
}