k8supdates: label nodes with k8s component hash

bootstrapper/internal/kubernetes/k8sapi/k8sutil.go

@@ -53,6 +53,7 @@ type Client interface {
 	AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string, namespace string) error
 	AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string, namespace string) error
 	ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error)
+	AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error
 }
 
 type installer interface {

bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go

@@ -67,6 +67,22 @@ func (k *Kubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMa
 	return nil
 }
 
+// AnnotateNode adds the provided annotations to the node, identified by name.
+func (k *Kubectl) AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error {
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		node, err := k.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		if node.Annotations == nil {
+			node.Annotations = map[string]string{}
+		}
+		node.Annotations[annotationKey] = annotationValue
+		_, err = k.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{})
+		return err
+	})
+}
+
 // ListAllNamespaces returns all namespaces in the cluster.
 func (k *Kubectl) ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error) {
 	return k.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})

bootstrapper/internal/kubernetes/kubeconfig.go

@@ -23,7 +23,7 @@ type KubeconfigReader struct {
 func (r KubeconfigReader) ReadKubeconfig() ([]byte, error) {
 	kubeconfig, err := r.fs.ReadFile(kubeconfigPath)
 	if err != nil {
-		return nil, fmt.Errorf("reading gce config: %w", err)
+		return nil, fmt.Errorf("reading kubernetes config: %w", err)
 	}
 	return kubeconfig, nil
 }

bootstrapper/internal/kubernetes/kubernetes.go

@@ -163,6 +163,13 @@ func (k *KubeWrapper) InitCluster(
 		return nil, fmt.Errorf("waiting for Kubernetes API to be available: %w", err)
 	}
 
+	// Annotate Node with the hash of the installed components
+	if err := k.client.AnnotateNode(ctx, nodeName,
+		constants.NodeKubernetesComponentsHashAnnotationKey, kubernetesComponents.GetHash(),
+	); err != nil {
+		return nil, fmt.Errorf("annotating node with Kubernetes components hash: %w", err)
+	}
+
 	// Step 3: configure & start kubernetes controllers
 	log.Infof("Starting Kubernetes controllers and deployments")
 	setupPodNetworkInput := k8sapi.SetupPodNetworkInput{

bootstrapper/internal/kubernetes/kubernetes_test.go

@@ -90,6 +90,25 @@ func TestInitCluster(t *testing.T) {
 			wantErr:    false,
 			k8sVersion: versions.Default,
 		},
+		"kubeadm init fails when annotating itself": {
+			clusterUtil: stubClusterUtil{},
+			kubeconfigReader: &stubKubeconfigReader{
+				kubeconfig: []byte("someKubeconfig"),
+			},
+			kubeAPIWaiter: stubKubeAPIWaiter{},
+			providerMetadata: &stubProviderMetadata{
+				selfResp: metadata.InstanceMetadata{
+					Name:          nodeName,
+					ProviderID:    providerID,
+					VPCIP:         privateIP,
+					AliasIPRanges: []string{aliasIPRange},
+				},
+				getLoadBalancerEndpointResp: loadbalancerIP,
+			},
+			kubectl:    stubKubectl{annotateNodeErr: someErr},
+			wantErr:    true,
+			k8sVersion: versions.Default,
+		},
 		"kubeadm init fails when retrieving metadata self": {
 			clusterUtil: stubClusterUtil{},
 			kubeconfigReader: &stubKubeconfigReader{
@@ -574,6 +593,7 @@ type stubKubectl struct {
 	addTNodeSelectorsToDeploymentErr error
 	waitForCRDsErr                   error
 	listAllNamespacesErr             error
+	annotateNodeErr                  error
 
 	listAllNamespacesResp *corev1.NamespaceList
 }
@@ -594,6 +614,10 @@ func (s *stubKubectl) AddNodeSelectorsToDeployment(ctx context.Context, selector
 	return s.addTNodeSelectorsToDeploymentErr
 }
 
+func (s *stubKubectl) AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error {
+	return s.annotateNodeErr
+}
+
 func (s *stubKubectl) WaitForCRDs(ctx context.Context, crds []string) error {
 	return s.waitForCRDsErr
 }

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/clusterrole.yaml

@@ -30,3 +30,12 @@ rules:
   - get
   - create
   - update
+- apiGroups:
+  - "update.edgeless.systems"
+  resources:
+  - joiningnodes
+  verbs:
+  - get
+  - create
+  - update
+  - patch

cli/internal/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml

@@ -0,0 +1,58 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: joiningnodes.update.edgeless.systems
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.9.0
+spec:
+  group: update.edgeless.systems
+  names:
+    kind: JoiningNode
+    listKind: JoiningNodeList
+    plural: joiningnodes
+    singular: joiningnode
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: JoiningNode is the Schema for the joiningnodes API.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: JoiningNodeSpec defines the components hash which the node
+              should be annotated with.
+            properties:
+              componentshash:
+                description: ComponentsHash is the hash of the components that were
+                  sent to the node by the join service.
+                type: string
+              name:
+                description: Name of the node expected to join.
+                type: string
+            type: object
+          status:
+            description: JoiningNodeStatus defines the observed state of JoiningNode.
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

cli/internal/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml

@@ -73,6 +73,32 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - update.edgeless.systems
+  resources:
+  - joiningnodes
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - update.edgeless.systems
+  resources:
+  - joiningnodes/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - update.edgeless.systems
+  resources:
+  - joiningnodes/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - update.edgeless.systems
   resources:

cli/internal/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml

@@ -76,6 +76,32 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - update.edgeless.systems
+  resources:
+  - joiningnodes
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - update.edgeless.systems
+  resources:
+  - joiningnodes/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - update.edgeless.systems
+  resources:
+  - joiningnodes/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - update.edgeless.systems
   resources:

cli/internal/helm/testdata/Azure/constellation-services/charts/join-service/templates/clusterrole.yaml

@@ -30,3 +30,12 @@ rules:
   - get
   - create
   - update
+- apiGroups:
+  - "update.edgeless.systems"
+  resources:
+  - joiningnodes
+  verbs:
+  - get
+  - create
+  - update
+  - patch

cli/internal/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml

@@ -76,6 +76,32 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - update.edgeless.systems
+  resources:
+  - joiningnodes
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - update.edgeless.systems
+  resources:
+  - joiningnodes/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - update.edgeless.systems
+  resources:
+  - joiningnodes/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - update.edgeless.systems
   resources:

cli/internal/helm/testdata/GCP/constellation-services/charts/join-service/templates/clusterrole.yaml

@@ -30,3 +30,12 @@ rules:
   - get
   - create
   - update
+- apiGroups:
+  - "update.edgeless.systems"
+  resources:
+  - joiningnodes
+  verbs:
+  - get
+  - create
+  - update
+  - patch

cli/internal/helm/testdata/QEMU/constellation-services/charts/join-service/templates/clusterrole.yaml

@@ -30,3 +30,12 @@ rules:
   - get
   - create
   - update
+- apiGroups:
+  - "update.edgeless.systems"
+  resources:
+  - joiningnodes
+  verbs:
+  - get
+  - create
+  - update
+  - patch

internal/constants/constants.go

@@ -124,6 +124,12 @@ const (
 	// ComponentsListKey is the name of the key holding the list of components in the components configMap.
 	ComponentsListKey = "components"
 
+	// NodeKubernetesComponentsHashAnnotationKey is the name of the annotation holding the hash of the installed components of this node.
+	NodeKubernetesComponentsHashAnnotationKey = "updates.edgeless.systems/kubernetes-components-hash"
+
+	// JoiningNodesConfigMapName is the name of the configMap holding the joining nodes with the components hashes the node-operator should annotate the nodes with.
+	JoiningNodesConfigMapName = "joining-nodes"
+
 	//
 	// CLI.
 	//

joinservice/internal/kubernetes/kubernetes.go

@@ -15,13 +15,17 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/versions"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 )
 
 // Client is a kubernetes client.
 type Client struct {
-	client *kubernetes.Clientset
+	client    *kubernetes.Clientset
+	dynClient dynamic.Interface
 }
 
 // New creates a new kubernetes client.
@@ -36,7 +40,13 @@ func New() (*Client, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create clientset: %w", err)
 	}
-	return &Client{client: clientset}, nil
+
+	dynClient, err := dynamic.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create dynamic client: %w", err)
+	}
+
+	return &Client{client: clientset, dynClient: dynClient}, nil
 }
 
 // GetComponents returns the components of the cluster.
@@ -70,6 +80,30 @@ func (c *Client) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap
 	return nil
 }
 
+// AddNodeToJoiningNodes adds the provided node as a joining node CRD.
+func (c *Client) AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string) error {
+	joiningNodeResource := schema.GroupVersionResource{Group: "update.edgeless.systems", Version: "v1alpha1", Resource: "joiningnodes"}
+
+	joiningNode := &unstructured.Unstructured{}
+	joiningNode.SetUnstructuredContent(map[string]any{
+		"apiVersion": "update.edgeless.systems/v1alpha1",
+		"kind":       "JoiningNode",
+		"metadata": map[string]any{
+			"name": nodeName,
+		},
+		"spec": map[string]any{
+			"name":           nodeName,
+			"componentshash": componentsHash,
+		},
+	})
+
+	_, err := c.dynClient.Resource(joiningNodeResource).Apply(ctx, joiningNode.GetName(), joiningNode, metav1.ApplyOptions{FieldManager: "join-service"})
+	if err != nil {
+		return fmt.Errorf("failed to create joining node: %w", err)
+	}
+	return nil
+}
+
 // AddReferenceToK8sVersionConfigMap adds a reference to the provided configmap to the k8s version configmap.
 func (c *Client) AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error {
 	cm, err := c.client.CoreV1().ConfigMaps("kube-system").Get(ctx, k8sVersionsConfigMapName, metav1.GetOptions{})

joinservice/internal/kubernetesca/kubernetesca.go

@@ -40,6 +40,19 @@ func New(log *logger.Logger, fileHandler file.Handler) *KubernetesCA {
 	}
 }
 
+// GetNodeNameFromCSR extracts the node name from a CSR.
+func (c KubernetesCA) GetNodeNameFromCSR(csr []byte) (string, error) {
+	certRequest, err := x509.ParseCertificateRequest(csr)
+	if err != nil {
+		return "", err
+	}
+	if !strings.HasPrefix(certRequest.Subject.CommonName, kubeconstants.NodesUserPrefix) {
+		return "", fmt.Errorf("certificate request must have common name prefix %q but is %q", kubeconstants.NodesUserPrefix, certRequest.Subject.CommonName)
+	}
+
+	return strings.TrimPrefix(certRequest.Subject.CommonName, kubeconstants.NodesUserPrefix), nil
+}
+
 // GetCertificate creates a certificate for a node and signs it using the Kubernetes root CA.
 func (c KubernetesCA) GetCertificate(csr []byte) (cert []byte, err error) {
 	c.log.Debugf("Loading Kubernetes CA certificate")

joinservice/internal/server/server.go

@@ -173,6 +173,15 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi
 		}
 	}
 
+	nodeName, err := s.ca.GetNodeNameFromCSR(req.CertificateRequest)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "unable to get node name from CSR: %s", err)
+	}
+
+	if err := s.kubeClient.AddNodeToJoiningNodes(ctx, nodeName, components.GetHash()); err != nil {
+		return nil, status.Errorf(codes.Internal, "unable to add node to joining nodes: %s", err)
+	}
+
 	log.Infof("IssueJoinTicket successful")
 	return &joinproto.IssueJoinTicketResponse{
 		StateDiskKey:             stateDiskKey,
@@ -294,10 +303,13 @@ type dataKeyGetter interface {
 type certificateAuthority interface {
 	// GetCertificate returns a certificate and private key, signed by the issuer.
 	GetCertificate(certificateRequest []byte) (kubeletCert []byte, err error)
+	// GetNodeNameFromCSR returns the node name from the CSR.
+	GetNodeNameFromCSR(csr []byte) (string, error)
 }
 
 type kubeClient interface {
 	GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error)
 	CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
+	AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string) error
 	AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error
 }

joinservice/internal/server/server_test.go

@@ -69,7 +69,7 @@ func TestIssueJoinTicket(t *testing.T) {
 				uuid:                                 testKey,
 				attestation.MeasurementSecretContext: measurementSecret,
 			}},
-			ca:         stubCA{cert: testCert},
+			ca:         stubCA{cert: testCert, nodeName: "node"},
 			kubeClient: stubKubeClient{getComponentsVal: components},
 		},
 		"worker node components reference missing": {
@@ -78,7 +78,7 @@ func TestIssueJoinTicket(t *testing.T) {
 				uuid:                                 testKey,
 				attestation.MeasurementSecretContext: measurementSecret,
 			}},
-			ca:                             stubCA{cert: testCert},
+			ca:                             stubCA{cert: testCert, nodeName: "node"},
 			kubeClient:                     stubKubeClient{getComponentsVal: components},
 			missingComponentsReferenceFile: true,
 		},
@@ -88,7 +88,7 @@ func TestIssueJoinTicket(t *testing.T) {
 				uuid:                                 testKey,
 				attestation.MeasurementSecretContext: measurementSecret,
 			}},
-			ca:                             stubCA{cert: testCert},
+			ca:                             stubCA{cert: testCert, nodeName: "node"},
 			kubeClient:                     stubKubeClient{createConfigMapErr: someErr},
 			missingComponentsReferenceFile: true,
 			wantErr:                        true,
@@ -99,14 +99,34 @@ func TestIssueJoinTicket(t *testing.T) {
 				uuid:                                 testKey,
 				attestation.MeasurementSecretContext: measurementSecret,
 			}},
-			ca:         stubCA{cert: testCert},
+			ca:         stubCA{cert: testCert, nodeName: "node"},
 			kubeClient: stubKubeClient{getComponentsErr: someErr},
 			wantErr:    true,
 		},
+		"Getting Node Name from CSR fails": {
+			kubeadm: stubTokenGetter{token: testJoinToken},
+			kms: stubKeyGetter{dataKeys: map[string][]byte{
+				uuid:                                 testKey,
+				attestation.MeasurementSecretContext: measurementSecret,
+			}},
+			ca:         stubCA{cert: testCert, nodeName: "node", getNameErr: someErr},
+			kubeClient: stubKubeClient{getComponentsVal: components},
+			wantErr:    true,
+		},
+		"Cannot add node to JoiningNode CRD": {
+			kubeadm: stubTokenGetter{token: testJoinToken},
+			kms: stubKeyGetter{dataKeys: map[string][]byte{
+				uuid:                                 testKey,
+				attestation.MeasurementSecretContext: measurementSecret,
+			}},
+			ca:         stubCA{cert: testCert, nodeName: "node"},
+			kubeClient: stubKubeClient{getComponentsVal: components, addNodeToJoiningNodesErr: someErr},
+			wantErr:    true,
+		},
 		"GetDataKey fails": {
 			kubeadm:    stubTokenGetter{token: testJoinToken},
 			kms:        stubKeyGetter{dataKeys: make(map[string][]byte), getDataKeyErr: someErr},
-			ca:         stubCA{cert: testCert},
+			ca:         stubCA{cert: testCert, nodeName: "node"},
 			kubeClient: stubKubeClient{getComponentsVal: components},
 			wantErr:    true,
 		},
@@ -116,7 +136,7 @@ func TestIssueJoinTicket(t *testing.T) {
 				uuid:                                 testKey,
 				attestation.MeasurementSecretContext: measurementSecret,
 			}},
-			ca:         stubCA{cert: testCert},
+			ca:         stubCA{cert: testCert, nodeName: "node"},
 			kubeClient: stubKubeClient{getComponentsVal: components},
 			wantErr:    true,
 		},
@@ -126,7 +146,7 @@ func TestIssueJoinTicket(t *testing.T) {
 				uuid:                                 testKey,
 				attestation.MeasurementSecretContext: measurementSecret,
 			}},
-			ca:         stubCA{getCertErr: someErr},
+			ca:         stubCA{getCertErr: someErr, nodeName: "node"},
 			kubeClient: stubKubeClient{getComponentsVal: components},
 			wantErr:    true,
 		},
@@ -140,7 +160,7 @@ func TestIssueJoinTicket(t *testing.T) {
 				uuid:                                 testKey,
 				attestation.MeasurementSecretContext: measurementSecret,
 			}},
-			ca:         stubCA{cert: testCert},
+			ca:         stubCA{cert: testCert, nodeName: "node"},
 			kubeClient: stubKubeClient{getComponentsVal: components},
 		},
 		"GetControlPlaneCertificateKey fails": {
@@ -150,7 +170,7 @@ func TestIssueJoinTicket(t *testing.T) {
 				uuid:                                 testKey,
 				attestation.MeasurementSecretContext: measurementSecret,
 			}},
-			ca:         stubCA{cert: testCert},
+			ca:         stubCA{cert: testCert, nodeName: "node"},
 			kubeClient: stubKubeClient{getComponentsVal: components},
 			wantErr:    true,
 		},
@@ -177,7 +197,7 @@ func TestIssueJoinTicket(t *testing.T) {
 				ca:              tc.ca,
 				joinTokenGetter: tc.kubeadm,
 				dataKeyGetter:   tc.kms,
-				kubeClient:      tc.kubeClient,
+				kubeClient:      &tc.kubeClient,
 				log:             logger.NewTest(t),
 			}
 
@@ -200,6 +220,8 @@ func TestIssueJoinTicket(t *testing.T) {
 			assert.Equal(tc.kubeadm.token.Token, resp.Token)
 			assert.Equal(tc.ca.cert, resp.KubeletCert)
 			assert.Equal(tc.kubeClient.getComponentsVal.ToJoinProto(), resp.KubernetesComponents)
+			assert.Equal(tc.ca.nodeName, tc.kubeClient.joiningNodeName)
+			assert.Equal(tc.kubeClient.getComponentsVal.GetHash(), tc.kubeClient.componentsHash)
 
 			if tc.isControlPlane {
 				assert.Len(resp.ControlPlaneFiles, len(tc.kubeadm.files))
@@ -288,29 +310,44 @@ func (f stubKeyGetter) GetDataKey(_ context.Context, name string, _ int) ([]byte
 type stubCA struct {
 	cert       []byte
 	getCertErr error
+	nodeName   string
+	getNameErr error
 }
 
 func (f stubCA) GetCertificate(csr []byte) ([]byte, error) {
 	return f.cert, f.getCertErr
 }
 
+func (f stubCA) GetNodeNameFromCSR(csr []byte) (string, error) {
+	return f.nodeName, f.getNameErr
+}
+
 type stubKubeClient struct {
 	getComponentsVal versions.ComponentVersions
 	getComponentsErr error
 
 	createConfigMapErr error
 
-	AddReferenceToK8sVersionConfigMapErr error
+	addReferenceToK8sVersionConfigMapErr error
+	addNodeToJoiningNodesErr             error
+	joiningNodeName                      string
+	componentsHash                       string
 }
 
-func (s stubKubeClient) GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error) {
+func (s *stubKubeClient) GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error) {
 	return s.getComponentsVal, s.getComponentsErr
 }
 
-func (s stubKubeClient) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
+func (s *stubKubeClient) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
 	return s.createConfigMapErr
 }
 
-func (s stubKubeClient) AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error {
-	return s.AddReferenceToK8sVersionConfigMapErr
+func (s *stubKubeClient) AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error {
+	return s.addReferenceToK8sVersionConfigMapErr
+}
+
+func (s *stubKubeClient) AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string) error {
+	s.joiningNodeName = nodeName
+	s.componentsHash = componentsHash
+	return s.addNodeToJoiningNodesErr
 }

operators/constellation-node-operator/.gitignore

@@ -16,6 +16,9 @@ testbin/*
 
 # Kubernetes Generated files - skip generated files, except for vendored files
 
+# We hold the charts in the cli/internal/helm directory
+chart/ 
+
 !vendor/**/zz_generated.*
 
 # editor and IDE paraphernalia

operators/constellation-node-operator/api/v1alpha1/joiningnodes_types.go

@@ -0,0 +1,48 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// JoiningNodeSpec defines the components hash which the node should be annotated with.
+type JoiningNodeSpec struct {
+	// Name of the node expected to join.
+	Name string `json:"name,omitempty"`
+	// ComponentsHash is the hash of the components that were sent to the node by the join service.
+	ComponentsHash string `json:"componentshash,omitempty"`
+}
+
+// JoiningNodeStatus defines the observed state of JoiningNode.
+type JoiningNodeStatus struct{}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:resource:scope=Cluster
+
+// JoiningNode is the Schema for the joiningnodes API.
+type JoiningNode struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   JoiningNodeSpec   `json:"spec,omitempty"`
+	Status JoiningNodeStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// JoiningNodeList contains a list of JoiningNodes.
+type JoiningNodeList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []JoiningNode `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&JoiningNode{}, &JoiningNodeList{})
+}

operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go

@@ -107,6 +107,95 @@ func (in *AutoscalingStrategyStatus) DeepCopy() *AutoscalingStrategyStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *JoiningNode) DeepCopyInto(out *JoiningNode) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	out.Spec = in.Spec
+	out.Status = in.Status
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoiningNode.
+func (in *JoiningNode) DeepCopy() *JoiningNode {
+	if in == nil {
+		return nil
+	}
+	out := new(JoiningNode)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *JoiningNode) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *JoiningNodeList) DeepCopyInto(out *JoiningNodeList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]JoiningNode, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoiningNodeList.
+func (in *JoiningNodeList) DeepCopy() *JoiningNodeList {
+	if in == nil {
+		return nil
+	}
+	out := new(JoiningNodeList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *JoiningNodeList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *JoiningNodeSpec) DeepCopyInto(out *JoiningNodeSpec) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoiningNodeSpec.
+func (in *JoiningNodeSpec) DeepCopy() *JoiningNodeSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(JoiningNodeSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *JoiningNodeStatus) DeepCopyInto(out *JoiningNodeStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoiningNodeStatus.
+func (in *JoiningNodeStatus) DeepCopy() *JoiningNodeStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(JoiningNodeStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NodeImage) DeepCopyInto(out *NodeImage) {
 	*out = *in

operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml

@@ -0,0 +1,54 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.9.0
+  creationTimestamp: null
+  name: joiningnodes.update.edgeless.systems
+spec:
+  group: update.edgeless.systems
+  names:
+    kind: JoiningNode
+    listKind: JoiningNodeList
+    plural: joiningnodes
+    singular: joiningnode
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: JoiningNode is the Schema for the joiningnodes API.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: JoiningNodeSpec defines the components hash which the node
+              should be annotated with.
+            properties:
+              componentshash:
+                description: ComponentsHash is the hash of the components that were
+                  sent to the node by the join service.
+                type: string
+              name:
+                description: Name of the node expected to join.
+                type: string
+            type: object
+          status:
+            description: JoiningNodeStatus defines the observed state of JoiningNode.
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

operators/constellation-node-operator/config/crd/kustomization.yaml

@@ -3,6 +3,7 @@
 # It should be run by config/default
 resources:
 - bases/update.edgeless.systems_nodeimages.yaml
+- bases/update.edgeless.systems_joiningnodes.yaml
 - bases/update.edgeless.systems_autoscalingstrategies.yaml
 - bases/update.edgeless.systems_scalinggroups.yaml
 - bases/update.edgeless.systems_pendingnodes.yaml
@@ -12,6 +13,7 @@ patchesStrategicMerge:
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
 # patches here are for enabling the conversion webhook for each CRD
 #- patches/webhook_in_nodeimages.yaml
+#- patches/webhook_in_joiningnodes.yaml
 #- patches/webhook_in_autoscalingstrategies.yaml
 #- patches/webhook_in_scalinggroups.yaml
 #- patches/webhook_in_pendingnodes.yaml
@@ -20,6 +22,7 @@ patchesStrategicMerge:
 # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix.
 # patches here are for enabling the CA injection for each CRD
 #- patches/cainjection_in_nodeimages.yaml
+#- patches/cainjection_in_joiningnodes.yaml
 #- patches/cainjection_in_autoscalingstrategies.yaml
 #- patches/cainjection_in_scalinggroups.yaml
 #- patches/cainjection_in_pendingnodes.yaml

operators/constellation-node-operator/config/crd/patches/cainjection_in_joiningnodes.yaml

@@ -0,0 +1,7 @@
+# The following patch adds a directive for certmanager to inject CA into the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+  name: joiningnodes.update.edgeless.systems

operators/constellation-node-operator/config/crd/patches/webhook_in_joiningnodes.yaml

@@ -0,0 +1,16 @@
+# The following patch enables a conversion webhook for the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: joiningnodes.update.edgeless.systems
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          namespace: system
+          name: webhook-service
+          path: /convert
+      conversionReviewVersions:
+      - v1

operators/constellation-node-operator/config/rbac/role.yaml

@@ -72,6 +72,32 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - update.edgeless.systems
+  resources:
+  - joiningnodes
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - update.edgeless.systems
+  resources:
+  - joiningnodes/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - update.edgeless.systems
+  resources:
+  - joiningnodes/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - update.edgeless.systems
   resources:

operators/constellation-node-operator/controllers/joiningnode_controller.go

@@ -0,0 +1,127 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package controllers
+
+import (
+	"context"
+
+	updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+const (
+	// NodeKubernetesComponentsHashAnnotationKey is the name of the annotation holding the hash of the installed components of this node.
+	NodeKubernetesComponentsHashAnnotationKey = "updates.edgeless.systems/kubernetes-components-hash"
+
+	joiningNodeNameKey = ".spec.name"
+)
+
+// JoiningNodesReconciler reconciles a JoiningNode object.
+type JoiningNodesReconciler struct {
+	client.Client
+	Scheme *runtime.Scheme
+}
+
+// NewJoiningNodesReconciler creates a new JoiningNodesReconciler.
+func NewJoiningNodesReconciler(client client.Client, scheme *runtime.Scheme) *JoiningNodesReconciler {
+	return &JoiningNodesReconciler{
+		Client: client,
+		Scheme: scheme,
+	}
+}
+
+//+kubebuilder:rbac:groups=update.edgeless.systems,resources=joiningnodes,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=update.edgeless.systems,resources=joiningnodes/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups=update.edgeless.systems,resources=joiningnodes/finalizers,verbs=update
+//+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch
+
+// Reconcile annotates the node with the components hash.
+func (r *JoiningNodesReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	logr := log.FromContext(ctx)
+
+	var joiningNode updatev1alpha1.JoiningNode
+	if err := r.Get(ctx, req.NamespacedName, &joiningNode); err != nil {
+		logr.Error(err, "unable to fetch JoiningNodes")
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		var node corev1.Node
+		if err := r.Get(ctx, types.NamespacedName{Name: joiningNode.Spec.Name}, &node); err != nil {
+			logr.Error(err, "unable to fetch Node")
+			return err
+		}
+
+		// add annotations to node
+		if node.Annotations == nil {
+			node.Annotations = map[string]string{}
+		}
+		node.Annotations[NodeKubernetesComponentsHashAnnotationKey] = joiningNode.Spec.ComponentsHash
+		return r.Update(ctx, &node)
+	})
+	if err != nil {
+		logr.Error(err, "unable to update Node")
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		if err := r.Delete(ctx, &joiningNode); err != nil {
+			logr.Error(err, "unable to delete JoiningNode")
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		logr.Error(err, "unable to delete JoiningNode")
+		return ctrl.Result{}, err
+	}
+
+	return ctrl.Result{}, nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *JoiningNodesReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	// index joining nodes by nodename.
+	if err := mgr.GetFieldIndexer().IndexField(context.Background(), &updatev1alpha1.JoiningNode{}, joiningNodeNameKey, func(rawObj client.Object) []string {
+		joiningNode := rawObj.(*updatev1alpha1.JoiningNode)
+		return []string{joiningNode.Spec.Name}
+	}); err != nil {
+		return err
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&updatev1alpha1.JoiningNode{}).
+		Watches(
+			&source.Kind{Type: &corev1.Node{}},
+			handler.EnqueueRequestsFromMapFunc(r.findAllJoiningNodes),
+		).
+		Complete(r)
+}
+
+func (r *JoiningNodesReconciler) findAllJoiningNodes(obj client.Object) []reconcile.Request {
+	var joiningNodesList updatev1alpha1.JoiningNodeList
+	err := r.List(context.TODO(), &joiningNodesList, client.MatchingFields{joiningNodeNameKey: obj.GetName()})
+	if err != nil {
+		return []reconcile.Request{}
+	}
+	requests := make([]reconcile.Request, len(joiningNodesList.Items))
+	for i, item := range joiningNodesList.Items {
+		requests[i] = reconcile.Request{
+			NamespacedName: types.NamespacedName{Name: item.GetName()},
+		}
+	}
+	return requests
+}

operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go

@@ -0,0 +1,143 @@
+//go:build integration
+
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package controllers
+
+import (
+	"context"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
+)
+
+var _ = Describe("JoiningNode controller", func() {
+	const (
+		nodeName1       = "node-name-1"
+		nodeName2       = "node-name-2"
+		componentsHash1 = "test-hash-1"
+		componentsHash2 = "test-hash-2"
+
+		timeout  = time.Second * 20
+		duration = time.Second * 2
+		interval = time.Millisecond * 250
+	)
+	Context("When changing a joining node resource spec", func() {
+		It("Should annotate the corresponding node when creating the CRD first", func() {
+			By("creating a joining node resource")
+			ctx := context.Background()
+			joiningNode := &updatev1alpha1.JoiningNode{
+				TypeMeta: metav1.TypeMeta{
+					APIVersion: "update.edgeless.systems/v1alpha1",
+					Kind:       "JoiningNode",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: nodeName1,
+				},
+				Spec: updatev1alpha1.JoiningNodeSpec{
+					Name:           nodeName1,
+					ComponentsHash: componentsHash1,
+				},
+			}
+			Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed())
+			createdJoiningNode := &updatev1alpha1.JoiningNode{}
+			Eventually(func() error {
+				return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdJoiningNode)
+			}, timeout, interval).Should(Succeed())
+			Expect(createdJoiningNode.Spec.Name).Should(Equal(nodeName1))
+			Expect(createdJoiningNode.Spec.ComponentsHash).Should(Equal(componentsHash1))
+
+			By("creating a node")
+			node := &corev1.Node{
+				TypeMeta: metav1.TypeMeta{
+					APIVersion: "update.edgeless.systems/v1alpha1",
+					Kind:       "Node",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: nodeName1,
+				},
+				Spec: corev1.NodeSpec{},
+			}
+			Expect(k8sClient.Create(ctx, node)).Should(Succeed())
+			createdNode := &corev1.Node{}
+			Eventually(func() error {
+				return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdNode)
+			}, timeout, interval).Should(Succeed())
+			Expect(createdNode.ObjectMeta.Name).Should(Equal(nodeName1))
+
+			By("annotating the node")
+			Eventually(func() string {
+				_ = k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdNode)
+				return createdNode.Annotations[NodeKubernetesComponentsHashAnnotationKey]
+			}, timeout, interval).Should(Equal(componentsHash1))
+
+			By("deleting the joining node resource")
+			Eventually(func() error {
+				return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode)
+			}, timeout, interval).ShouldNot(Succeed())
+		})
+	})
+	It("Should annotate the corresponding node when creating the node first", func() {
+		ctx := context.Background()
+		By("creating a node")
+		node := &corev1.Node{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: "update.edgeless.systems/v1alpha1",
+				Kind:       "Node",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name: nodeName2,
+			},
+			Spec: corev1.NodeSpec{},
+		}
+		Expect(k8sClient.Create(ctx, node)).Should(Succeed())
+		createdNode := &corev1.Node{}
+		Eventually(func() error {
+			return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName2}, createdNode)
+		}, timeout, interval).Should(Succeed())
+		Expect(createdNode.ObjectMeta.Name).Should(Equal(nodeName2))
+
+		By("creating a joining node resource")
+		joiningNode := &updatev1alpha1.JoiningNode{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: "update.edgeless.systems/v1alpha1",
+				Kind:       "JoiningNode",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name: nodeName2,
+			},
+			Spec: updatev1alpha1.JoiningNodeSpec{
+				Name:           nodeName2,
+				ComponentsHash: componentsHash2,
+			},
+		}
+		Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed())
+		createdJoiningNode := &updatev1alpha1.JoiningNode{}
+		Eventually(func() error {
+			return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode)
+		}, timeout, interval).Should(Succeed())
+		Expect(createdJoiningNode.Spec.Name).Should(Equal(nodeName2))
+		Expect(createdJoiningNode.Spec.ComponentsHash).Should(Equal(componentsHash2))
+
+		By("annotating the node")
+		Eventually(func() string {
+			_ = k8sClient.Get(ctx, types.NamespacedName{Name: createdNode.Name}, createdNode)
+			return createdNode.Annotations[NodeKubernetesComponentsHashAnnotationKey]
+		}, timeout, interval).Should(Equal(componentsHash2))
+
+		By("deleting the joining node resource")
+		Eventually(func() error {
+			return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode)
+		}, timeout, interval).ShouldNot(Succeed())
+	})
+})

operators/constellation-node-operator/controllers/suite_test.go

@@ -93,6 +93,12 @@ var _ = BeforeSuite(func() {
 	}).SetupWithManager(k8sManager)
 	Expect(err).ToNot(HaveOccurred())
 
+	err = (&JoiningNodesReconciler{
+		Client: k8sManager.GetClient(),
+		Scheme: k8sManager.GetScheme(),
+	}).SetupWithManager(k8sManager)
+	Expect(err).ToNot(HaveOccurred())
+
 	err = (&ScalingGroupReconciler{
 		scalingGroupUpdater: fakes.scalingGroupUpdater,
 		Client:              k8sManager.GetClient(),

operators/constellation-node-operator/main.go

@@ -145,6 +145,13 @@ func main() {
 		setupLog.Error(err, "Unable to create controller", "controller", "AutoscalingStrategy")
 		os.Exit(1)
 	}
+	if err = (&controllers.JoiningNodesReconciler{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "Unable to create controller", "controller", "JoiningNode")
+		os.Exit(1)
+	}
 	if err = controllers.NewScalingGroupReconciler(
 		cspClient, mgr.GetClient(), mgr.GetScheme(),
 	).SetupWithManager(mgr); err != nil {