Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-07-20 05:51:46 -04:00
Code Issues Projects Releases Packages Wiki Activity

upgrade: support Kubernetes components (#839)

Browse source 
* upgrade: add Kubernetes components to NodeVersion

* update rfc
This commit is contained in:
3u13r 2023-01-03 12:09:53 +01:00 • committed by GitHub
parent 4b43311fbd
commit f14af0c3eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
56 changed files with 897 additions and 738 deletions
Download patch file Download diff file Expand all files Collapse all files

1
joinservice/cmd/main.go
View file

@ -87,7 +87,6 @@ func main() {
server, err := server.New(
measurementSalt,
handler,
kubernetesca.New(log.Named("certificateAuthority"), handler),
kubeadm,
kms,

28
joinservice/internal/kubernetes/kubernetes.go
View file

@ -73,8 +73,26 @@ func (c *Client) getConfigMapData(ctx context.Context, name, key string) (string
return cm.Data[key], nil
}
// GetK8sComponentsRefFromNodeVersionCRD returns the K8sComponentsRef from the node version CRD.
func (c *Client) GetK8sComponentsRefFromNodeVersionCRD(ctx context.Context, nodeName string) (string, error) {
nodeVersionResource := schema.GroupVersionResource{Group: "update.edgeless.systems", Version: "v1alpha1", Resource: "nodeversions"}
nodeVersion, err := c.dynClient.Resource(nodeVersionResource).Get(ctx, nodeName, metav1.GetOptions{})
if err != nil {
return "", fmt.Errorf("failed to get node version: %w", err)
}
// Extract K8sComponentsRef from nodeVersion.
k8sComponentsRef, found, err := unstructured.NestedString(nodeVersion.Object, "spec", "kubernetesComponentsReference")
if err != nil {
return "", fmt.Errorf("failed to get K8sComponentsRef from node version: %w", err)
}
if !found {
return "", fmt.Errorf("kubernetesComponentsReference not found in node version")
}
return k8sComponentsRef, nil
}
// AddNodeToJoiningNodes adds the provided node as a joining node CRD.
func (c *Client) AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string, isControlPlane bool) error {
func (c *Client) AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsReference string, isControlPlane bool) error {
joiningNode := &unstructured.Unstructured{}
compliantNodeName, err := k8sCompliantHostname(nodeName)
@ -98,10 +116,10 @@ func (c *Client) AddNodeToJoiningNodes(ctx context.Context, nodeName string, com
"name": objectMetadataName,
},
"spec": map[string]any{
"name": compliantNodeName,
"componentshash": componentsHash,
"iscontrolplane": isControlPlane,
"deadline": deadline,
"name": compliantNodeName,
"componentsreference": componentsReference,
"iscontrolplane": isControlPlane,
"deadline": deadline,
},
})
if isControlPlane {

41
joinservice/internal/server/server.go
View file

@ -10,13 +10,11 @@ import (
"context"
"fmt"
"net"
"path/filepath"
"time"
"github.com/edgelesssys/constellation/v2/internal/attestation"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/crypto"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/grpclog"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/versions"
@ -36,7 +34,6 @@ type Server struct {
measurementSalt []byte
log *logger.Logger
file file.Handler
joinTokenGetter joinTokenGetter
dataKeyGetter dataKeyGetter
ca certificateAuthority
@ -46,7 +43,7 @@ type Server struct {
// New initializes a new Server.
func New(
measurementSalt []byte, fileHandler file.Handler, ca certificateAuthority,
measurementSalt []byte, ca certificateAuthority,
joinTokenGetter joinTokenGetter, dataKeyGetter dataKeyGetter, log *logger.Logger,
) (*Server, error) {
kubeClient, err := kubernetes.New()
@ -56,7 +53,6 @@ func New(
return &Server{
measurementSalt: measurementSalt,
log: log,
file: fileHandler,
joinTokenGetter: joinTokenGetter,
dataKeyGetter: dataKeyGetter,
ca: ca,
@ -114,14 +110,8 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi
return nil, status.Errorf(codes.Internal, "unable to generate Kubernetes join arguments: %s", err)
}
log.Infof("Querying K8sVersion ConfigMap for Kubernetes version")
k8sVersion, err := s.getK8sVersion()
if err != nil {
return nil, status.Errorf(codes.Internal, "unable to get k8s version: %s", err)
}
log.Infof("Querying K8sVersion ConfigMap for components ConfigMap name")
componentsConfigMapName, err := s.getK8sComponentsConfigMapName()
log.Infof("Querying NodeVersion CR for components ConfigMap name")
componentsConfigMapName, err := s.getK8sComponentsConfigMapName(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "unable to get components ConfigMap name: %s", err)
}
@ -160,7 +150,7 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi
return nil, status.Errorf(codes.Internal, "unable to get node name from CSR: %s", err)
}
if err := s.kubeClient.AddNodeToJoiningNodes(ctx, nodeName, components.GetHash(), req.IsControlPlane); err != nil {
if err := s.kubeClient.AddNodeToJoiningNodes(ctx, nodeName, componentsConfigMapName, req.IsControlPlane); err != nil {
return nil, status.Errorf(codes.Internal, "unable to add node to joining nodes: %s", err)
}
@ -174,7 +164,6 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi
DiscoveryTokenCaCertHash: kubeArgs.CACertHashes[0],
KubeletCert: kubeletCert,
ControlPlaneFiles: controlPlaneFiles,
KubernetesVersion: k8sVersion,
KubernetesComponents: components.ToJoinProto(),
}, nil
}
@ -204,26 +193,13 @@ func (s *Server) IssueRejoinTicket(ctx context.Context, req *joinproto.IssueRejo
}, nil
}
// getK8sVersion reads the k8s version from a VolumeMount that is backed by the k8s-version ConfigMap.
func (s *Server) getK8sVersion() (string, error) {
fileContent, err := s.file.Read(filepath.Join(constants.ServiceBasePath, constants.K8sVersionConfigMapName))
if err != nil {
return "", fmt.Errorf("could not read k8s version file: %w", err)
}
k8sVersion := string(fileContent)
return k8sVersion, nil
}
// getK8sComponentsConfigMapName reads the k8s components config map name from a VolumeMount that is backed by the k8s-version ConfigMap.
func (s *Server) getK8sComponentsConfigMapName() (string, error) {
fileContent, err := s.file.Read(filepath.Join(constants.ServiceBasePath, constants.K8sComponentsFieldName))
func (s *Server) getK8sComponentsConfigMapName(ctx context.Context) (string, error) {
k8sComponentsRef, err := s.kubeClient.GetK8sComponentsRefFromNodeVersionCRD(ctx, "constellation-version")
if err != nil {
return "", fmt.Errorf("could not read k8s version file: %w", err)
return "", fmt.Errorf("could not get k8s components config map name: %w", err)
}
componentsConfigMapName := string(fileContent)
return componentsConfigMapName, nil
return k8sComponentsRef, nil
}
// joinTokenGetter returns Kubernetes bootstrap (join) tokens.
@ -247,6 +223,7 @@ type certificateAuthority interface {
}
type kubeClient interface {
GetK8sComponentsRefFromNodeVersionCRD(ctx context.Context, nodeName string) (string, error)
GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error)
AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string, isControlPlane bool) error
}

57
joinservice/internal/server/server_test.go
View file

@ -9,17 +9,13 @@ package server
import (
"context"
"errors"
"path/filepath"
"testing"
"time"
"github.com/edgelesssys/constellation/v2/internal/attestation"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/edgelesssys/constellation/v2/joinservice/joinproto"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
@ -42,7 +38,6 @@ func TestIssueJoinTicket(t *testing.T) {
CACertHashes: []string{"hash"},
Token: "token",
}
testK8sVersion := versions.Default
components := versions.ComponentVersions{
{
@ -69,18 +64,7 @@ func TestIssueJoinTicket(t *testing.T) {
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
},
"worker node components reference missing": {
kubeadm: stubTokenGetter{token: testJoinToken},
kms: stubKeyGetter{dataKeys: map[string][]byte{
uuid: testKey,
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
missingComponentsReferenceFile: true,
wantErr: true,
kubeClient: stubKubeClient{getComponentsVal: components, getK8sComponentsRefFromNodeVersionCRDVal: "k8s-components-ref"},
},
"kubeclient fails": {
kubeadm: stubTokenGetter{token: testJoinToken},
@ -99,7 +83,7 @@ func TestIssueJoinTicket(t *testing.T) {
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert, nodeName: "node", getNameErr: someErr},
kubeClient: stubKubeClient{getComponentsVal: components},
kubeClient: stubKubeClient{getComponentsVal: components, getK8sComponentsRefFromNodeVersionCRDVal: "k8s-components-ref"},
wantErr: true,
},
"Cannot add node to JoiningNode CRD": {
@ -109,14 +93,14 @@ func TestIssueJoinTicket(t *testing.T) {
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components, addNodeToJoiningNodesErr: someErr},
kubeClient: stubKubeClient{getComponentsVal: components, addNodeToJoiningNodesErr: someErr, getK8sComponentsRefFromNodeVersionCRDVal: "k8s-components-ref"},
wantErr: true,
},
"GetDataKey fails": {
kubeadm: stubTokenGetter{token: testJoinToken},
kms: stubKeyGetter{dataKeys: make(map[string][]byte), getDataKeyErr: someErr},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
kubeClient: stubKubeClient{getComponentsVal: components, getK8sComponentsRefFromNodeVersionCRDVal: "k8s-components-ref"},
wantErr: true,
},
"GetJoinToken fails": {
@ -126,7 +110,7 @@ func TestIssueJoinTicket(t *testing.T) {
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
kubeClient: stubKubeClient{getComponentsVal: components, getK8sComponentsRefFromNodeVersionCRDVal: "k8s-components-ref"},
wantErr: true,
},
"GetCertificate fails": {
@ -136,7 +120,7 @@ func TestIssueJoinTicket(t *testing.T) {
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{getCertErr: someErr, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
kubeClient: stubKubeClient{getComponentsVal: components, getK8sComponentsRefFromNodeVersionCRDVal: "k8s-components-ref"},
wantErr: true,
},
"control plane": {
@ -150,7 +134,7 @@ func TestIssueJoinTicket(t *testing.T) {
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
kubeClient: stubKubeClient{getComponentsVal: components, getK8sComponentsRefFromNodeVersionCRDVal: "k8s-components-ref"},
},
"GetControlPlaneCertificateKey fails": {
isControlPlane: true,
@ -160,7 +144,7 @@ func TestIssueJoinTicket(t *testing.T) {
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
kubeClient: stubKubeClient{getComponentsVal: components, getK8sComponentsRefFromNodeVersionCRDVal: "k8s-components-ref"},
wantErr: true,
},
}
@ -170,19 +154,10 @@ func TestIssueJoinTicket(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
handler := file.NewHandler(afero.NewMemMapFs())
// IssueJoinTicket tries to read the k8s-version ConfigMap from a mounted file.
require.NoError(handler.Write(filepath.Join(constants.ServiceBasePath, constants.K8sVersionConfigMapName), []byte(testK8sVersion), file.OptNone))
if !tc.missingComponentsReferenceFile {
require.NoError(handler.Write(filepath.Join(constants.ServiceBasePath, constants.K8sComponentsFieldName), []byte(testK8sVersion), file.OptNone))
}
salt := []byte{0xA, 0xB, 0xC}
api := Server{
measurementSalt: salt,
file: handler,
ca: tc.ca,
joinTokenGetter: tc.kubeadm,
dataKeyGetter: tc.kms,
@ -210,7 +185,7 @@ func TestIssueJoinTicket(t *testing.T) {
assert.Equal(tc.ca.cert, resp.KubeletCert)
assert.Equal(tc.kubeClient.getComponentsVal.ToJoinProto(), resp.KubernetesComponents)
assert.Equal(tc.ca.nodeName, tc.kubeClient.joiningNodeName)
assert.Equal(tc.kubeClient.getComponentsVal.GetHash(), tc.kubeClient.componentsHash)
assert.Equal(tc.kubeClient.getK8sComponentsRefFromNodeVersionCRDVal, tc.kubeClient.componentsRef)
if tc.isControlPlane {
assert.Len(resp.ControlPlaneFiles, len(tc.kubeadm.files))
@ -249,7 +224,6 @@ func TestIssueRejoinTicker(t *testing.T) {
require := require.New(t)
api := Server{
file: file.Handler{},
ca: stubCA{},
joinTokenGetter: stubTokenGetter{},
dataKeyGetter: tc.keyGetter,
@ -315,17 +289,24 @@ type stubKubeClient struct {
getComponentsVal versions.ComponentVersions
getComponentsErr error
getK8sComponentsRefFromNodeVersionCRDErr error
getK8sComponentsRefFromNodeVersionCRDVal string
addNodeToJoiningNodesErr error
joiningNodeName string
componentsHash string
componentsRef string
}
func (s *stubKubeClient) GetK8sComponentsRefFromNodeVersionCRD(ctx context.Context, nodeName string) (string, error) {
return s.getK8sComponentsRefFromNodeVersionCRDVal, s.getK8sComponentsRefFromNodeVersionCRDErr
}
func (s *stubKubeClient) GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error) {
return s.getComponentsVal, s.getComponentsErr
}
func (s *stubKubeClient) AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string, isControlPlane bool) error {
func (s *stubKubeClient) AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsRef string, isControlPlane bool) error {
s.joiningNodeName = nodeName
s.componentsHash = componentsHash
s.componentsRef = componentsRef
return s.addNodeToJoiningNodesErr
}