Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-10-01 01:36:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

k8supdates: label nodes with k8s component hash

Browse Source
This commit is contained in:
Leonard Cohnen 2022-12-06 18:48:01 +01:00 committed by 3u13r
parent 1466c12972
commit a1161ae05d
30 changed files with 869 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bootstrapper/internal/kubernetes/k8sapi/k8sutil.go
View File

@ -53,6 +53,7 @@ type Client interface {
AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string, namespace string) error
AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string, namespace string) error
ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error)
AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error
}
type installer interface {

16
bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go
View File

@ -67,6 +67,22 @@ func (k *Kubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMa
return nil
}
// AnnotateNode adds the provided annotations to the node, identified by name.
func (k *Kubectl) AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
node, err := k.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
if err != nil {
return err
}
if node.Annotations == nil {
node.Annotations = map[string]string{}
}
node.Annotations[annotationKey] = annotationValue
_, err = k.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{})
return err
})
}
// ListAllNamespaces returns all namespaces in the cluster.
func (k *Kubectl) ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error) {
return k.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})

2
bootstrapper/internal/kubernetes/kubeconfig.go
View File

@ -23,7 +23,7 @@ type KubeconfigReader struct {
func (r KubeconfigReader) ReadKubeconfig() ([]byte, error) {
kubeconfig, err := r.fs.ReadFile(kubeconfigPath)
if err != nil {
return nil, fmt.Errorf("reading gce config: %w", err)
return nil, fmt.Errorf("reading kubernetes config: %w", err)
}
return kubeconfig, nil
}

7
bootstrapper/internal/kubernetes/kubernetes.go
View File

@ -163,6 +163,13 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("waiting for Kubernetes API to be available: %w", err)
}
// Annotate Node with the hash of the installed components
if err := k.client.AnnotateNode(ctx, nodeName,
constants.NodeKubernetesComponentsHashAnnotationKey, kubernetesComponents.GetHash(),
); err != nil {
return nil, fmt.Errorf("annotating node with Kubernetes components hash: %w", err)
}
// Step 3: configure & start kubernetes controllers
log.Infof("Starting Kubernetes controllers and deployments")
setupPodNetworkInput := k8sapi.SetupPodNetworkInput{

24
bootstrapper/internal/kubernetes/kubernetes_test.go
View File

@ -90,6 +90,25 @@ func TestInitCluster(t *testing.T) {
wantErr: false,
k8sVersion: versions.Default,
},
"kubeadm init fails when annotating itself": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
kubeconfig: []byte("someKubeconfig"),
},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{
selfResp: metadata.InstanceMetadata{
Name: nodeName,
ProviderID: providerID,
VPCIP: privateIP,
AliasIPRanges: []string{aliasIPRange},
},
getLoadBalancerEndpointResp: loadbalancerIP,
},
kubectl: stubKubectl{annotateNodeErr: someErr},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when retrieving metadata self": {
clusterUtil: stubClusterUtil{},
kubeconfigReader: &stubKubeconfigReader{
@ -574,6 +593,7 @@ type stubKubectl struct {
addTNodeSelectorsToDeploymentErr error
waitForCRDsErr error
listAllNamespacesErr error
annotateNodeErr error
listAllNamespacesResp *corev1.NamespaceList
}
@ -594,6 +614,10 @@ func (s *stubKubectl) AddNodeSelectorsToDeployment(ctx context.Context, selector
return s.addTNodeSelectorsToDeploymentErr
}
func (s *stubKubectl) AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error {
return s.annotateNodeErr
}
func (s *stubKubectl) WaitForCRDs(ctx context.Context, crds []string) error {
return s.waitForCRDsErr
}

9
cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/clusterrole.yaml
View File

@ -30,3 +30,12 @@ rules:
- get
- create
- update
- apiGroups:
- "update.edgeless.systems"
resources:
- joiningnodes
verbs:
- get
- create
- update
- patch

58
cli/internal/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml Normal file
View File

@ -0,0 +1,58 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: joiningnodes.update.edgeless.systems
annotations:
controller-gen.kubebuilder.io/version: v0.9.0
spec:
group: update.edgeless.systems
names:
kind: JoiningNode
listKind: JoiningNodeList
plural: joiningnodes
singular: joiningnode
scope: Cluster
versions:
- name: v1alpha1
schema:
openAPIV3Schema:
description: JoiningNode is the Schema for the joiningnodes API.
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
description: JoiningNodeSpec defines the components hash which the node
should be annotated with.
properties:
componentshash:
description: ComponentsHash is the hash of the components that were
sent to the node by the join service.
type: string
name:
description: Name of the node expected to join.
type: string
type: object
status:
description: JoiningNodeStatus defines the observed state of JoiningNode.
type: object
type: object
served: true
storage: true
subresources:
status: {}
status:
acceptedNames:
kind: ""
plural: ""
conditions: []
storedVersions: []

26
cli/internal/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml
View File

@ -73,6 +73,32 @@ rules:
- get
- patch
- update
- apiGroups:
- update.edgeless.systems
resources:
- joiningnodes
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- update.edgeless.systems
resources:
- joiningnodes/finalizers
verbs:
- update
- apiGroups:
- update.edgeless.systems
resources:
- joiningnodes/status
verbs:
- get
- patch
- update
- apiGroups:
- update.edgeless.systems
resources:

26
cli/internal/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml vendored
View File

@ -76,6 +76,32 @@ rules:
- get
- patch
- update
- apiGroups:
- update.edgeless.systems
resources:
- joiningnodes
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- update.edgeless.systems
resources:
- joiningnodes/finalizers
verbs:
- update
- apiGroups:
- update.edgeless.systems
resources:
- joiningnodes/status
verbs:
- get
- patch
- update
- apiGroups:
- update.edgeless.systems
resources:

9
cli/internal/helm/testdata/Azure/constellation-services/charts/join-service/templates/clusterrole.yaml vendored
View File

@ -30,3 +30,12 @@ rules:
- get
- create
- update
- apiGroups:
- "update.edgeless.systems"
resources:
- joiningnodes
verbs:
- get
- create
- update
- patch

26
cli/internal/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml vendored
View File

@ -76,6 +76,32 @@ rules:
- get
- patch
- update
- apiGroups:
- update.edgeless.systems
resources:
- joiningnodes
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- update.edgeless.systems
resources:
- joiningnodes/finalizers
verbs:
- update
- apiGroups:
- update.edgeless.systems
resources:
- joiningnodes/status
verbs:
- get
- patch
- update
- apiGroups:
- update.edgeless.systems
resources:

9
cli/internal/helm/testdata/GCP/constellation-services/charts/join-service/templates/clusterrole.yaml vendored
View File

@ -30,3 +30,12 @@ rules:
- get
- create
- update
- apiGroups:
- "update.edgeless.systems"
resources:
- joiningnodes
verbs:
- get
- create
- update
- patch

9
cli/internal/helm/testdata/QEMU/constellation-services/charts/join-service/templates/clusterrole.yaml vendored
View File

@ -30,3 +30,12 @@ rules:
- get
- create
- update
- apiGroups:
- "update.edgeless.systems"
resources:
- joiningnodes
verbs:
- get
- create
- update
- patch

6
internal/constants/constants.go
View File

@ -124,6 +124,12 @@ const (
// ComponentsListKey is the name of the key holding the list of components in the components configMap.
ComponentsListKey = "components"
// NodeKubernetesComponentsHashAnnotationKey is the name of the annotation holding the hash of the installed components of this node.
NodeKubernetesComponentsHashAnnotationKey = "updates.edgeless.systems/kubernetes-components-hash"
// JoiningNodesConfigMapName is the name of the configMap holding the joining nodes with the components hashes the node-operator should annotate the nodes with.
JoiningNodesConfigMapName = "joining-nodes"
//
// CLI.
//

38
joinservice/internal/kubernetes/kubernetes.go
View File

@ -15,13 +15,17 @@ import (
"github.com/edgelesssys/constellation/v2/internal/versions"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
// Client is a kubernetes client.
type Client struct {
client *kubernetes.Clientset
client *kubernetes.Clientset
dynClient dynamic.Interface
}
// New creates a new kubernetes client.
@ -36,7 +40,13 @@ func New() (*Client, error) {
if err != nil {
return nil, fmt.Errorf("failed to create clientset: %w", err)
}
return &Client{client: clientset}, nil
dynClient, err := dynamic.NewForConfig(config)
if err != nil {
return nil, fmt.Errorf("failed to create dynamic client: %w", err)
}
return &Client{client: clientset, dynClient: dynClient}, nil
}
// GetComponents returns the components of the cluster.
@ -70,6 +80,30 @@ func (c *Client) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap
return nil
}
// AddNodeToJoiningNodes adds the provided node as a joining node CRD.
func (c *Client) AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string) error {
joiningNodeResource := schema.GroupVersionResource{Group: "update.edgeless.systems", Version: "v1alpha1", Resource: "joiningnodes"}
joiningNode := &unstructured.Unstructured{}
joiningNode.SetUnstructuredContent(map[string]any{
"apiVersion": "update.edgeless.systems/v1alpha1",
"kind": "JoiningNode",
"metadata": map[string]any{
"name": nodeName,
},
"spec": map[string]any{
"name": nodeName,
"componentshash": componentsHash,
},
})
_, err := c.dynClient.Resource(joiningNodeResource).Apply(ctx, joiningNode.GetName(), joiningNode, metav1.ApplyOptions{FieldManager: "join-service"})
if err != nil {
return fmt.Errorf("failed to create joining node: %w", err)
}
return nil
}
// AddReferenceToK8sVersionConfigMap adds a reference to the provided configmap to the k8s version configmap.
func (c *Client) AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error {
cm, err := c.client.CoreV1().ConfigMaps("kube-system").Get(ctx, k8sVersionsConfigMapName, metav1.GetOptions{})

13
joinservice/internal/kubernetesca/kubernetesca.go
View File

@ -40,6 +40,19 @@ func New(log *logger.Logger, fileHandler file.Handler) *KubernetesCA {
}
}
// GetNodeNameFromCSR extracts the node name from a CSR.
func (c KubernetesCA) GetNodeNameFromCSR(csr []byte) (string, error) {
certRequest, err := x509.ParseCertificateRequest(csr)
if err != nil {
return "", err
}
if !strings.HasPrefix(certRequest.Subject.CommonName, kubeconstants.NodesUserPrefix) {
return "", fmt.Errorf("certificate request must have common name prefix %q but is %q", kubeconstants.NodesUserPrefix, certRequest.Subject.CommonName)
}
return strings.TrimPrefix(certRequest.Subject.CommonName, kubeconstants.NodesUserPrefix), nil
}
// GetCertificate creates a certificate for a node and signs it using the Kubernetes root CA.
func (c KubernetesCA) GetCertificate(csr []byte) (cert []byte, err error) {
c.log.Debugf("Loading Kubernetes CA certificate")

12
joinservice/internal/server/server.go
View File

@ -173,6 +173,15 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi
}
}
nodeName, err := s.ca.GetNodeNameFromCSR(req.CertificateRequest)
if err != nil {
return nil, status.Errorf(codes.Internal, "unable to get node name from CSR: %s", err)
}
if err := s.kubeClient.AddNodeToJoiningNodes(ctx, nodeName, components.GetHash()); err != nil {
return nil, status.Errorf(codes.Internal, "unable to add node to joining nodes: %s", err)
}
log.Infof("IssueJoinTicket successful")
return &joinproto.IssueJoinTicketResponse{
StateDiskKey: stateDiskKey,
@ -294,10 +303,13 @@ type dataKeyGetter interface {
type certificateAuthority interface {
// GetCertificate returns a certificate and private key, signed by the issuer.
GetCertificate(certificateRequest []byte) (kubeletCert []byte, err error)
// GetNodeNameFromCSR returns the node name from the CSR.
GetNodeNameFromCSR(csr []byte) (string, error)
}
type kubeClient interface {
GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error)
CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string) error
AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error
}

67
joinservice/internal/server/server_test.go
View File

@ -69,7 +69,7 @@ func TestIssueJoinTicket(t *testing.T) {
uuid: testKey,
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
},
"worker node components reference missing": {
@ -78,7 +78,7 @@ func TestIssueJoinTicket(t *testing.T) {
uuid: testKey,
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
missingComponentsReferenceFile: true,
},
@ -88,7 +88,7 @@ func TestIssueJoinTicket(t *testing.T) {
uuid: testKey,
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{createConfigMapErr: someErr},
missingComponentsReferenceFile: true,
wantErr: true,
@ -99,14 +99,34 @@ func TestIssueJoinTicket(t *testing.T) {
uuid: testKey,
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsErr: someErr},
wantErr: true,
},
"Getting Node Name from CSR fails": {
kubeadm: stubTokenGetter{token: testJoinToken},
kms: stubKeyGetter{dataKeys: map[string][]byte{
uuid: testKey,
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert, nodeName: "node", getNameErr: someErr},
kubeClient: stubKubeClient{getComponentsVal: components},
wantErr: true,
},
"Cannot add node to JoiningNode CRD": {
kubeadm: stubTokenGetter{token: testJoinToken},
kms: stubKeyGetter{dataKeys: map[string][]byte{
uuid: testKey,
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components, addNodeToJoiningNodesErr: someErr},
wantErr: true,
},
"GetDataKey fails": {
kubeadm: stubTokenGetter{token: testJoinToken},
kms: stubKeyGetter{dataKeys: make(map[string][]byte), getDataKeyErr: someErr},
ca: stubCA{cert: testCert},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
wantErr: true,
},
@ -116,7 +136,7 @@ func TestIssueJoinTicket(t *testing.T) {
uuid: testKey,
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
wantErr: true,
},
@ -126,7 +146,7 @@ func TestIssueJoinTicket(t *testing.T) {
uuid: testKey,
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{getCertErr: someErr},
ca: stubCA{getCertErr: someErr, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
wantErr: true,
},
@ -140,7 +160,7 @@ func TestIssueJoinTicket(t *testing.T) {
uuid: testKey,
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
},
"GetControlPlaneCertificateKey fails": {
@ -150,7 +170,7 @@ func TestIssueJoinTicket(t *testing.T) {
uuid: testKey,
attestation.MeasurementSecretContext: measurementSecret,
}},
ca: stubCA{cert: testCert},
ca: stubCA{cert: testCert, nodeName: "node"},
kubeClient: stubKubeClient{getComponentsVal: components},
wantErr: true,
},
@ -177,7 +197,7 @@ func TestIssueJoinTicket(t *testing.T) {
ca: tc.ca,
joinTokenGetter: tc.kubeadm,
dataKeyGetter: tc.kms,
kubeClient: tc.kubeClient,
kubeClient: &tc.kubeClient,
log: logger.NewTest(t),
}
@ -200,6 +220,8 @@ func TestIssueJoinTicket(t *testing.T) {
assert.Equal(tc.kubeadm.token.Token, resp.Token)
assert.Equal(tc.ca.cert, resp.KubeletCert)
assert.Equal(tc.kubeClient.getComponentsVal.ToJoinProto(), resp.KubernetesComponents)
assert.Equal(tc.ca.nodeName, tc.kubeClient.joiningNodeName)
assert.Equal(tc.kubeClient.getComponentsVal.GetHash(), tc.kubeClient.componentsHash)
if tc.isControlPlane {
assert.Len(resp.ControlPlaneFiles, len(tc.kubeadm.files))
@ -288,29 +310,44 @@ func (f stubKeyGetter) GetDataKey(_ context.Context, name string, _ int) ([]byte
type stubCA struct {
cert []byte
getCertErr error
nodeName string
getNameErr error
}
func (f stubCA) GetCertificate(csr []byte) ([]byte, error) {
return f.cert, f.getCertErr
}
func (f stubCA) GetNodeNameFromCSR(csr []byte) (string, error) {
return f.nodeName, f.getNameErr
}
type stubKubeClient struct {
getComponentsVal versions.ComponentVersions
getComponentsErr error
createConfigMapErr error
AddReferenceToK8sVersionConfigMapErr error
addReferenceToK8sVersionConfigMapErr error
addNodeToJoiningNodesErr error
joiningNodeName string
componentsHash string
}
func (s stubKubeClient) GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error) {
func (s *stubKubeClient) GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error) {
return s.getComponentsVal, s.getComponentsErr
}
func (s stubKubeClient) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
func (s *stubKubeClient) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
return s.createConfigMapErr
}
func (s stubKubeClient) AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error {
return s.AddReferenceToK8sVersionConfigMapErr
func (s *stubKubeClient) AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error {
return s.addReferenceToK8sVersionConfigMapErr
}
func (s *stubKubeClient) AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string) error {
s.joiningNodeName = nodeName
s.componentsHash = componentsHash
return s.addNodeToJoiningNodesErr
}

3
operators/constellation-node-operator/.gitignore vendored
View File

@ -16,6 +16,9 @@ testbin/*
# Kubernetes Generated files - skip generated files, except for vendored files
# We hold the charts in the cli/internal/helm directory
chart/
!vendor/**/zz_generated.*
# editor and IDE paraphernalia

48
operators/constellation-node-operator/api/v1alpha1/joiningnodes_types.go Normal file
View File

@ -0,0 +1,48 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// JoiningNodeSpec defines the components hash which the node should be annotated with.
type JoiningNodeSpec struct {
// Name of the node expected to join.
Name string `json:"name,omitempty"`
// ComponentsHash is the hash of the components that were sent to the node by the join service.
ComponentsHash string `json:"componentshash,omitempty"`
}
// JoiningNodeStatus defines the observed state of JoiningNode.
type JoiningNodeStatus struct{}
//+kubebuilder:object:root=true
//+kubebuilder:subresource:status
//+kubebuilder:resource:scope=Cluster
// JoiningNode is the Schema for the joiningnodes API.
type JoiningNode struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec JoiningNodeSpec `json:"spec,omitempty"`
Status JoiningNodeStatus `json:"status,omitempty"`
}
//+kubebuilder:object:root=true
// JoiningNodeList contains a list of JoiningNodes.
type JoiningNodeList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []JoiningNode `json:"items"`
}
func init() {
SchemeBuilder.Register(&JoiningNode{}, &JoiningNodeList{})
}

89
operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go
View File

@ -107,6 +107,95 @@ func (in *AutoscalingStrategyStatus) DeepCopy() *AutoscalingStrategyStatus {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *JoiningNode) DeepCopyInto(out *JoiningNode) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
out.Spec = in.Spec
out.Status = in.Status
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoiningNode.
func (in *JoiningNode) DeepCopy() *JoiningNode {
if in == nil {
return nil
}
out := new(JoiningNode)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *JoiningNode) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *JoiningNodeList) DeepCopyInto(out *JoiningNodeList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]JoiningNode, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoiningNodeList.
func (in *JoiningNodeList) DeepCopy() *JoiningNodeList {
if in == nil {
return nil
}
out := new(JoiningNodeList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *JoiningNodeList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *JoiningNodeSpec) DeepCopyInto(out *JoiningNodeSpec) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoiningNodeSpec.
func (in *JoiningNodeSpec) DeepCopy() *JoiningNodeSpec {
if in == nil {
return nil
}
out := new(JoiningNodeSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *JoiningNodeStatus) DeepCopyInto(out *JoiningNodeStatus) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoiningNodeStatus.
func (in *JoiningNodeStatus) DeepCopy() *JoiningNodeStatus {
if in == nil {
return nil
}
out := new(JoiningNodeStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeImage) DeepCopyInto(out *NodeImage) {
*out = *in

54
operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml Normal file
View File

@ -0,0 +1,54 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.9.0
creationTimestamp: null
name: joiningnodes.update.edgeless.systems
spec:
group: update.edgeless.systems
names:
kind: JoiningNode
listKind: JoiningNodeList
plural: joiningnodes
singular: joiningnode
scope: Cluster
versions:
- name: v1alpha1
schema:
openAPIV3Schema:
description: JoiningNode is the Schema for the joiningnodes API.
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
description: JoiningNodeSpec defines the components hash which the node
should be annotated with.
properties:
componentshash:
description: ComponentsHash is the hash of the components that were
sent to the node by the join service.
type: string
name:
description: Name of the node expected to join.
type: string
type: object
status:
description: JoiningNodeStatus defines the observed state of JoiningNode.
type: object
type: object
served: true
storage: true
subresources:
status: {}

3
operators/constellation-node-operator/config/crd/kustomization.yaml
View File

@ -3,6 +3,7 @@
# It should be run by config/default
resources:
- bases/update.edgeless.systems_nodeimages.yaml
- bases/update.edgeless.systems_joiningnodes.yaml
- bases/update.edgeless.systems_autoscalingstrategies.yaml
- bases/update.edgeless.systems_scalinggroups.yaml
- bases/update.edgeless.systems_pendingnodes.yaml
@ -12,6 +13,7 @@ patchesStrategicMerge:
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
# patches here are for enabling the conversion webhook for each CRD
#- patches/webhook_in_nodeimages.yaml
#- patches/webhook_in_joiningnodes.yaml
#- patches/webhook_in_autoscalingstrategies.yaml
#- patches/webhook_in_scalinggroups.yaml
#- patches/webhook_in_pendingnodes.yaml
@ -20,6 +22,7 @@ patchesStrategicMerge:
# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix.
# patches here are for enabling the CA injection for each CRD
#- patches/cainjection_in_nodeimages.yaml
#- patches/cainjection_in_joiningnodes.yaml
#- patches/cainjection_in_autoscalingstrategies.yaml
#- patches/cainjection_in_scalinggroups.yaml
#- patches/cainjection_in_pendingnodes.yaml

7
operators/constellation-node-operator/config/crd/patches/cainjection_in_joiningnodes.yaml Normal file
View File

@ -0,0 +1,7 @@
# The following patch adds a directive for certmanager to inject CA into the CRD
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
name: joiningnodes.update.edgeless.systems

16
operators/constellation-node-operator/config/crd/patches/webhook_in_joiningnodes.yaml Normal file
View File

@ -0,0 +1,16 @@
# The following patch enables a conversion webhook for the CRD
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: joiningnodes.update.edgeless.systems
spec:
conversion:
strategy: Webhook
webhook:
clientConfig:
service:
namespace: system
name: webhook-service
path: /convert
conversionReviewVersions:
- v1

26
operators/constellation-node-operator/config/rbac/role.yaml
View File

@ -72,6 +72,32 @@ rules:
- get
- patch
- update
- apiGroups:
- update.edgeless.systems
resources:
- joiningnodes
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- update.edgeless.systems
resources:
- joiningnodes/finalizers
verbs:
- update
- apiGroups:
- update.edgeless.systems
resources:
- joiningnodes/status
verbs:
- get
- patch
- update
- apiGroups:
- update.edgeless.systems
resources:

127
operators/constellation-node-operator/controllers/joiningnode_controller.go Normal file
View File

@ -0,0 +1,127 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package controllers
import (
"context"
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
)
const (
// NodeKubernetesComponentsHashAnnotationKey is the name of the annotation holding the hash of the installed components of this node.
NodeKubernetesComponentsHashAnnotationKey = "updates.edgeless.systems/kubernetes-components-hash"
joiningNodeNameKey = ".spec.name"
)
// JoiningNodesReconciler reconciles a JoiningNode object.
type JoiningNodesReconciler struct {
client.Client
Scheme *runtime.Scheme
}
// NewJoiningNodesReconciler creates a new JoiningNodesReconciler.
func NewJoiningNodesReconciler(client client.Client, scheme *runtime.Scheme) *JoiningNodesReconciler {
return &JoiningNodesReconciler{
Client: client,
Scheme: scheme,
}
}
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=joiningnodes,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=joiningnodes/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=joiningnodes/finalizers,verbs=update
//+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch
// Reconcile annotates the node with the components hash.
func (r *JoiningNodesReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logr := log.FromContext(ctx)
var joiningNode updatev1alpha1.JoiningNode
if err := r.Get(ctx, req.NamespacedName, &joiningNode); err != nil {
logr.Error(err, "unable to fetch JoiningNodes")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
var node corev1.Node
if err := r.Get(ctx, types.NamespacedName{Name: joiningNode.Spec.Name}, &node); err != nil {
logr.Error(err, "unable to fetch Node")
return err
}
// add annotations to node
if node.Annotations == nil {
node.Annotations = map[string]string{}
}
node.Annotations[NodeKubernetesComponentsHashAnnotationKey] = joiningNode.Spec.ComponentsHash
return r.Update(ctx, &node)
})
if err != nil {
logr.Error(err, "unable to update Node")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
if err := r.Delete(ctx, &joiningNode); err != nil {
logr.Error(err, "unable to delete JoiningNode")
return err
}
return nil
})
if err != nil {
logr.Error(err, "unable to delete JoiningNode")
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *JoiningNodesReconciler) SetupWithManager(mgr ctrl.Manager) error {
// index joining nodes by nodename.
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &updatev1alpha1.JoiningNode{}, joiningNodeNameKey, func(rawObj client.Object) []string {
joiningNode := rawObj.(*updatev1alpha1.JoiningNode)
return []string{joiningNode.Spec.Name}
}); err != nil {
return err
}
return ctrl.NewControllerManagedBy(mgr).
For(&updatev1alpha1.JoiningNode{}).
Watches(
&source.Kind{Type: &corev1.Node{}},
handler.EnqueueRequestsFromMapFunc(r.findAllJoiningNodes),
).
Complete(r)
}
func (r *JoiningNodesReconciler) findAllJoiningNodes(obj client.Object) []reconcile.Request {
var joiningNodesList updatev1alpha1.JoiningNodeList
err := r.List(context.TODO(), &joiningNodesList, client.MatchingFields{joiningNodeNameKey: obj.GetName()})
if err != nil {
return []reconcile.Request{}
}
requests := make([]reconcile.Request, len(joiningNodesList.Items))
for i, item := range joiningNodesList.Items {
requests[i] = reconcile.Request{
NamespacedName: types.NamespacedName{Name: item.GetName()},
}
}
return requests
}

143
operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go Normal file
View File

@ -0,0 +1,143 @@
//go:build integration
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package controllers
import (
"context"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
)
var _ = Describe("JoiningNode controller", func() {
const (
nodeName1 = "node-name-1"
nodeName2 = "node-name-2"
componentsHash1 = "test-hash-1"
componentsHash2 = "test-hash-2"
timeout = time.Second * 20
duration = time.Second * 2
interval = time.Millisecond * 250
)
Context("When changing a joining node resource spec", func() {
It("Should annotate the corresponding node when creating the CRD first", func() {
By("creating a joining node resource")
ctx := context.Background()
joiningNode := &updatev1alpha1.JoiningNode{
TypeMeta: metav1.TypeMeta{
APIVersion: "update.edgeless.systems/v1alpha1",
Kind: "JoiningNode",
},
ObjectMeta: metav1.ObjectMeta{
Name: nodeName1,
},
Spec: updatev1alpha1.JoiningNodeSpec{
Name: nodeName1,
ComponentsHash: componentsHash1,
},
}
Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed())
createdJoiningNode := &updatev1alpha1.JoiningNode{}
Eventually(func() error {
return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdJoiningNode)
}, timeout, interval).Should(Succeed())
Expect(createdJoiningNode.Spec.Name).Should(Equal(nodeName1))
Expect(createdJoiningNode.Spec.ComponentsHash).Should(Equal(componentsHash1))
By("creating a node")
node := &corev1.Node{
TypeMeta: metav1.TypeMeta{
APIVersion: "update.edgeless.systems/v1alpha1",
Kind: "Node",
},
ObjectMeta: metav1.ObjectMeta{
Name: nodeName1,
},
Spec: corev1.NodeSpec{},
}
Expect(k8sClient.Create(ctx, node)).Should(Succeed())
createdNode := &corev1.Node{}
Eventually(func() error {
return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdNode)
}, timeout, interval).Should(Succeed())
Expect(createdNode.ObjectMeta.Name).Should(Equal(nodeName1))
By("annotating the node")
Eventually(func() string {
_ = k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdNode)
return createdNode.Annotations[NodeKubernetesComponentsHashAnnotationKey]
}, timeout, interval).Should(Equal(componentsHash1))
By("deleting the joining node resource")
Eventually(func() error {
return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode)
}, timeout, interval).ShouldNot(Succeed())
})
})
It("Should annotate the corresponding node when creating the node first", func() {
ctx := context.Background()
By("creating a node")
node := &corev1.Node{
TypeMeta: metav1.TypeMeta{
APIVersion: "update.edgeless.systems/v1alpha1",
Kind: "Node",
},
ObjectMeta: metav1.ObjectMeta{
Name: nodeName2,
},
Spec: corev1.NodeSpec{},
}
Expect(k8sClient.Create(ctx, node)).Should(Succeed())
createdNode := &corev1.Node{}
Eventually(func() error {
return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName2}, createdNode)
}, timeout, interval).Should(Succeed())
Expect(createdNode.ObjectMeta.Name).Should(Equal(nodeName2))
By("creating a joining node resource")
joiningNode := &updatev1alpha1.JoiningNode{
TypeMeta: metav1.TypeMeta{
APIVersion: "update.edgeless.systems/v1alpha1",
Kind: "JoiningNode",
},
ObjectMeta: metav1.ObjectMeta{
Name: nodeName2,
},
Spec: updatev1alpha1.JoiningNodeSpec{
Name: nodeName2,
ComponentsHash: componentsHash2,
},
}
Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed())
createdJoiningNode := &updatev1alpha1.JoiningNode{}
Eventually(func() error {
return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode)
}, timeout, interval).Should(Succeed())
Expect(createdJoiningNode.Spec.Name).Should(Equal(nodeName2))
Expect(createdJoiningNode.Spec.ComponentsHash).Should(Equal(componentsHash2))
By("annotating the node")
Eventually(func() string {
_ = k8sClient.Get(ctx, types.NamespacedName{Name: createdNode.Name}, createdNode)
return createdNode.Annotations[NodeKubernetesComponentsHashAnnotationKey]
}, timeout, interval).Should(Equal(componentsHash2))
By("deleting the joining node resource")
Eventually(func() error {
return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode)
}, timeout, interval).ShouldNot(Succeed())
})
})

6
operators/constellation-node-operator/controllers/suite_test.go
View File

@ -93,6 +93,12 @@ var _ = BeforeSuite(func() {
}).SetupWithManager(k8sManager)
Expect(err).ToNot(HaveOccurred())
err = (&JoiningNodesReconciler{
Client: k8sManager.GetClient(),
Scheme: k8sManager.GetScheme(),
}).SetupWithManager(k8sManager)
Expect(err).ToNot(HaveOccurred())
err = (&ScalingGroupReconciler{
scalingGroupUpdater: fakes.scalingGroupUpdater,
Client: k8sManager.GetClient(),

7
operators/constellation-node-operator/main.go
View File

@ -145,6 +145,13 @@ func main() {
setupLog.Error(err, "Unable to create controller", "controller", "AutoscalingStrategy")
os.Exit(1)
}
if err = (&controllers.JoiningNodesReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "Unable to create controller", "controller", "JoiningNode")
os.Exit(1)
}
if err = controllers.NewScalingGroupReconciler(
cspClient, mgr.GetClient(), mgr.GetScheme(),
).SetupWithManager(mgr); err != nil {