From a1161ae05d284c6309622f60b85d89b0cbfc3f6a Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Tue, 6 Dec 2022 18:48:01 +0100 Subject: [PATCH] k8supdates: label nodes with k8s component hash --- .../internal/kubernetes/k8sapi/k8sutil.go | 1 + .../kubernetes/k8sapi/kubectl/kubectl.go | 16 ++ .../internal/kubernetes/kubeconfig.go | 2 +- .../internal/kubernetes/kubernetes.go | 7 + .../internal/kubernetes/kubernetes_test.go | 24 +++ .../join-service/templates/clusterrole.yaml | 9 ++ .../crds/joiningnode-crd.yaml | 58 +++++++ .../templates/manager-rbac.yaml | 26 ++++ .../templates/manager-rbac.yaml | 26 ++++ .../join-service/templates/clusterrole.yaml | 9 ++ .../templates/manager-rbac.yaml | 26 ++++ .../join-service/templates/clusterrole.yaml | 9 ++ .../join-service/templates/clusterrole.yaml | 9 ++ internal/constants/constants.go | 6 + joinservice/internal/kubernetes/kubernetes.go | 38 ++++- .../internal/kubernetesca/kubernetesca.go | 13 ++ joinservice/internal/server/server.go | 12 ++ joinservice/internal/server/server_test.go | 67 ++++++-- .../constellation-node-operator/.gitignore | 3 + .../api/v1alpha1/joiningnodes_types.go | 48 ++++++ .../api/v1alpha1/zz_generated.deepcopy.go | 89 +++++++++++ .../update.edgeless.systems_joiningnodes.yaml | 54 +++++++ .../config/crd/kustomization.yaml | 3 + .../patches/cainjection_in_joiningnodes.yaml | 7 + .../crd/patches/webhook_in_joiningnodes.yaml | 16 ++ .../config/rbac/role.yaml | 26 ++++ .../controllers/joiningnode_controller.go | 127 ++++++++++++++++ .../joiningnode_controller_env_test.go | 143 ++++++++++++++++++ .../controllers/suite_test.go | 6 + operators/constellation-node-operator/main.go | 7 + 30 files changed, 869 insertions(+), 18 deletions(-) create mode 100644 cli/internal/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml create mode 100644 operators/constellation-node-operator/api/v1alpha1/joiningnodes_types.go create mode 100644 operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml create mode 100644 operators/constellation-node-operator/config/crd/patches/cainjection_in_joiningnodes.yaml create mode 100644 operators/constellation-node-operator/config/crd/patches/webhook_in_joiningnodes.yaml create mode 100644 operators/constellation-node-operator/controllers/joiningnode_controller.go create mode 100644 operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go diff --git a/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go b/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go index ff6195ea2..8d3cebdb8 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go +++ b/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go @@ -53,6 +53,7 @@ type Client interface { AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string, namespace string) error AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string, namespace string) error ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error) + AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error } type installer interface { diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go b/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go index 523458c9e..91d7eac95 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go @@ -67,6 +67,22 @@ func (k *Kubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMa return nil } +// AnnotateNode adds the provided annotations to the node, identified by name. +func (k *Kubectl) AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + node, err := k.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + node.Annotations[annotationKey] = annotationValue + _, err = k.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) + return err + }) +} + // ListAllNamespaces returns all namespaces in the cluster. func (k *Kubectl) ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error) { return k.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) diff --git a/bootstrapper/internal/kubernetes/kubeconfig.go b/bootstrapper/internal/kubernetes/kubeconfig.go index d55e5f3f6..4d70ba19f 100644 --- a/bootstrapper/internal/kubernetes/kubeconfig.go +++ b/bootstrapper/internal/kubernetes/kubeconfig.go @@ -23,7 +23,7 @@ type KubeconfigReader struct { func (r KubeconfigReader) ReadKubeconfig() ([]byte, error) { kubeconfig, err := r.fs.ReadFile(kubeconfigPath) if err != nil { - return nil, fmt.Errorf("reading gce config: %w", err) + return nil, fmt.Errorf("reading kubernetes config: %w", err) } return kubeconfig, nil } diff --git a/bootstrapper/internal/kubernetes/kubernetes.go b/bootstrapper/internal/kubernetes/kubernetes.go index 92ebc0814..e2609cf4d 100644 --- a/bootstrapper/internal/kubernetes/kubernetes.go +++ b/bootstrapper/internal/kubernetes/kubernetes.go @@ -163,6 +163,13 @@ func (k *KubeWrapper) InitCluster( return nil, fmt.Errorf("waiting for Kubernetes API to be available: %w", err) } + // Annotate Node with the hash of the installed components + if err := k.client.AnnotateNode(ctx, nodeName, + constants.NodeKubernetesComponentsHashAnnotationKey, kubernetesComponents.GetHash(), + ); err != nil { + return nil, fmt.Errorf("annotating node with Kubernetes components hash: %w", err) + } + // Step 3: configure & start kubernetes controllers log.Infof("Starting Kubernetes controllers and deployments") setupPodNetworkInput := k8sapi.SetupPodNetworkInput{ diff --git a/bootstrapper/internal/kubernetes/kubernetes_test.go b/bootstrapper/internal/kubernetes/kubernetes_test.go index c700aa578..e60ccda57 100644 --- a/bootstrapper/internal/kubernetes/kubernetes_test.go +++ b/bootstrapper/internal/kubernetes/kubernetes_test.go @@ -90,6 +90,25 @@ func TestInitCluster(t *testing.T) { wantErr: false, k8sVersion: versions.Default, }, + "kubeadm init fails when annotating itself": { + clusterUtil: stubClusterUtil{}, + kubeconfigReader: &stubKubeconfigReader{ + kubeconfig: []byte("someKubeconfig"), + }, + kubeAPIWaiter: stubKubeAPIWaiter{}, + providerMetadata: &stubProviderMetadata{ + selfResp: metadata.InstanceMetadata{ + Name: nodeName, + ProviderID: providerID, + VPCIP: privateIP, + AliasIPRanges: []string{aliasIPRange}, + }, + getLoadBalancerEndpointResp: loadbalancerIP, + }, + kubectl: stubKubectl{annotateNodeErr: someErr}, + wantErr: true, + k8sVersion: versions.Default, + }, "kubeadm init fails when retrieving metadata self": { clusterUtil: stubClusterUtil{}, kubeconfigReader: &stubKubeconfigReader{ @@ -574,6 +593,7 @@ type stubKubectl struct { addTNodeSelectorsToDeploymentErr error waitForCRDsErr error listAllNamespacesErr error + annotateNodeErr error listAllNamespacesResp *corev1.NamespaceList } @@ -594,6 +614,10 @@ func (s *stubKubectl) AddNodeSelectorsToDeployment(ctx context.Context, selector return s.addTNodeSelectorsToDeploymentErr } +func (s *stubKubectl) AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error { + return s.annotateNodeErr +} + func (s *stubKubectl) WaitForCRDs(ctx context.Context, crds []string) error { return s.waitForCRDsErr } diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/clusterrole.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/clusterrole.yaml index 64f4fc143..cbf7efd3f 100644 --- a/cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/clusterrole.yaml +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/clusterrole.yaml @@ -30,3 +30,12 @@ rules: - get - create - update +- apiGroups: + - "update.edgeless.systems" + resources: + - joiningnodes + verbs: + - get + - create + - update + - patch diff --git a/cli/internal/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml b/cli/internal/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml new file mode 100644 index 000000000..a7b8ffaa0 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml @@ -0,0 +1,58 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: joiningnodes.update.edgeless.systems + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 +spec: + group: update.edgeless.systems + names: + kind: JoiningNode + listKind: JoiningNodeList + plural: joiningnodes + singular: joiningnode + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: JoiningNode is the Schema for the joiningnodes API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: JoiningNodeSpec defines the components hash which the node + should be annotated with. + properties: + componentshash: + description: ComponentsHash is the hash of the components that were + sent to the node by the join service. + type: string + name: + description: Name of the node expected to join. + type: string + type: object + status: + description: JoiningNodeStatus defines the observed state of JoiningNode. + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] \ No newline at end of file diff --git a/cli/internal/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml b/cli/internal/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml index 7a749c591..c276d8c18 100644 --- a/cli/internal/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/cli/internal/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -73,6 +73,32 @@ rules: - get - patch - update +- apiGroups: + - update.edgeless.systems + resources: + - joiningnodes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - update.edgeless.systems + resources: + - joiningnodes/finalizers + verbs: + - update +- apiGroups: + - update.edgeless.systems + resources: + - joiningnodes/status + verbs: + - get + - patch + - update - apiGroups: - update.edgeless.systems resources: diff --git a/cli/internal/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/cli/internal/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index f0c447f95..1a1b3e64a 100644 --- a/cli/internal/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/cli/internal/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -76,6 +76,32 @@ rules: - get - patch - update +- apiGroups: + - update.edgeless.systems + resources: + - joiningnodes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - update.edgeless.systems + resources: + - joiningnodes/finalizers + verbs: + - update +- apiGroups: + - update.edgeless.systems + resources: + - joiningnodes/status + verbs: + - get + - patch + - update - apiGroups: - update.edgeless.systems resources: diff --git a/cli/internal/helm/testdata/Azure/constellation-services/charts/join-service/templates/clusterrole.yaml b/cli/internal/helm/testdata/Azure/constellation-services/charts/join-service/templates/clusterrole.yaml index 64f4fc143..517902a3b 100644 --- a/cli/internal/helm/testdata/Azure/constellation-services/charts/join-service/templates/clusterrole.yaml +++ b/cli/internal/helm/testdata/Azure/constellation-services/charts/join-service/templates/clusterrole.yaml @@ -30,3 +30,12 @@ rules: - get - create - update +- apiGroups: + - "update.edgeless.systems" + resources: + - joiningnodes + verbs: + - get + - create + - update + - patch \ No newline at end of file diff --git a/cli/internal/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/cli/internal/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index f0c447f95..1a1b3e64a 100644 --- a/cli/internal/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/cli/internal/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -76,6 +76,32 @@ rules: - get - patch - update +- apiGroups: + - update.edgeless.systems + resources: + - joiningnodes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - update.edgeless.systems + resources: + - joiningnodes/finalizers + verbs: + - update +- apiGroups: + - update.edgeless.systems + resources: + - joiningnodes/status + verbs: + - get + - patch + - update - apiGroups: - update.edgeless.systems resources: diff --git a/cli/internal/helm/testdata/GCP/constellation-services/charts/join-service/templates/clusterrole.yaml b/cli/internal/helm/testdata/GCP/constellation-services/charts/join-service/templates/clusterrole.yaml index 64f4fc143..cbf7efd3f 100644 --- a/cli/internal/helm/testdata/GCP/constellation-services/charts/join-service/templates/clusterrole.yaml +++ b/cli/internal/helm/testdata/GCP/constellation-services/charts/join-service/templates/clusterrole.yaml @@ -30,3 +30,12 @@ rules: - get - create - update +- apiGroups: + - "update.edgeless.systems" + resources: + - joiningnodes + verbs: + - get + - create + - update + - patch diff --git a/cli/internal/helm/testdata/QEMU/constellation-services/charts/join-service/templates/clusterrole.yaml b/cli/internal/helm/testdata/QEMU/constellation-services/charts/join-service/templates/clusterrole.yaml index 64f4fc143..517902a3b 100644 --- a/cli/internal/helm/testdata/QEMU/constellation-services/charts/join-service/templates/clusterrole.yaml +++ b/cli/internal/helm/testdata/QEMU/constellation-services/charts/join-service/templates/clusterrole.yaml @@ -30,3 +30,12 @@ rules: - get - create - update +- apiGroups: + - "update.edgeless.systems" + resources: + - joiningnodes + verbs: + - get + - create + - update + - patch \ No newline at end of file diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 4ed61f9b7..3590195fa 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -124,6 +124,12 @@ const ( // ComponentsListKey is the name of the key holding the list of components in the components configMap. ComponentsListKey = "components" + // NodeKubernetesComponentsHashAnnotationKey is the name of the annotation holding the hash of the installed components of this node. + NodeKubernetesComponentsHashAnnotationKey = "updates.edgeless.systems/kubernetes-components-hash" + + // JoiningNodesConfigMapName is the name of the configMap holding the joining nodes with the components hashes the node-operator should annotate the nodes with. + JoiningNodesConfigMapName = "joining-nodes" + // // CLI. // diff --git a/joinservice/internal/kubernetes/kubernetes.go b/joinservice/internal/kubernetes/kubernetes.go index 3a403a06f..41666ad57 100644 --- a/joinservice/internal/kubernetes/kubernetes.go +++ b/joinservice/internal/kubernetes/kubernetes.go @@ -15,13 +15,17 @@ import ( "github.com/edgelesssys/constellation/v2/internal/versions" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) // Client is a kubernetes client. type Client struct { - client *kubernetes.Clientset + client *kubernetes.Clientset + dynClient dynamic.Interface } // New creates a new kubernetes client. @@ -36,7 +40,13 @@ func New() (*Client, error) { if err != nil { return nil, fmt.Errorf("failed to create clientset: %w", err) } - return &Client{client: clientset}, nil + + dynClient, err := dynamic.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create dynamic client: %w", err) + } + + return &Client{client: clientset, dynClient: dynClient}, nil } // GetComponents returns the components of the cluster. @@ -70,6 +80,30 @@ func (c *Client) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap return nil } +// AddNodeToJoiningNodes adds the provided node as a joining node CRD. +func (c *Client) AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string) error { + joiningNodeResource := schema.GroupVersionResource{Group: "update.edgeless.systems", Version: "v1alpha1", Resource: "joiningnodes"} + + joiningNode := &unstructured.Unstructured{} + joiningNode.SetUnstructuredContent(map[string]any{ + "apiVersion": "update.edgeless.systems/v1alpha1", + "kind": "JoiningNode", + "metadata": map[string]any{ + "name": nodeName, + }, + "spec": map[string]any{ + "name": nodeName, + "componentshash": componentsHash, + }, + }) + + _, err := c.dynClient.Resource(joiningNodeResource).Apply(ctx, joiningNode.GetName(), joiningNode, metav1.ApplyOptions{FieldManager: "join-service"}) + if err != nil { + return fmt.Errorf("failed to create joining node: %w", err) + } + return nil +} + // AddReferenceToK8sVersionConfigMap adds a reference to the provided configmap to the k8s version configmap. func (c *Client) AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error { cm, err := c.client.CoreV1().ConfigMaps("kube-system").Get(ctx, k8sVersionsConfigMapName, metav1.GetOptions{}) diff --git a/joinservice/internal/kubernetesca/kubernetesca.go b/joinservice/internal/kubernetesca/kubernetesca.go index 8dc3e67f3..e8405a881 100644 --- a/joinservice/internal/kubernetesca/kubernetesca.go +++ b/joinservice/internal/kubernetesca/kubernetesca.go @@ -40,6 +40,19 @@ func New(log *logger.Logger, fileHandler file.Handler) *KubernetesCA { } } +// GetNodeNameFromCSR extracts the node name from a CSR. +func (c KubernetesCA) GetNodeNameFromCSR(csr []byte) (string, error) { + certRequest, err := x509.ParseCertificateRequest(csr) + if err != nil { + return "", err + } + if !strings.HasPrefix(certRequest.Subject.CommonName, kubeconstants.NodesUserPrefix) { + return "", fmt.Errorf("certificate request must have common name prefix %q but is %q", kubeconstants.NodesUserPrefix, certRequest.Subject.CommonName) + } + + return strings.TrimPrefix(certRequest.Subject.CommonName, kubeconstants.NodesUserPrefix), nil +} + // GetCertificate creates a certificate for a node and signs it using the Kubernetes root CA. func (c KubernetesCA) GetCertificate(csr []byte) (cert []byte, err error) { c.log.Debugf("Loading Kubernetes CA certificate") diff --git a/joinservice/internal/server/server.go b/joinservice/internal/server/server.go index 297bb6248..6ecd88e3f 100644 --- a/joinservice/internal/server/server.go +++ b/joinservice/internal/server/server.go @@ -173,6 +173,15 @@ func (s *Server) IssueJoinTicket(ctx context.Context, req *joinproto.IssueJoinTi } } + nodeName, err := s.ca.GetNodeNameFromCSR(req.CertificateRequest) + if err != nil { + return nil, status.Errorf(codes.Internal, "unable to get node name from CSR: %s", err) + } + + if err := s.kubeClient.AddNodeToJoiningNodes(ctx, nodeName, components.GetHash()); err != nil { + return nil, status.Errorf(codes.Internal, "unable to add node to joining nodes: %s", err) + } + log.Infof("IssueJoinTicket successful") return &joinproto.IssueJoinTicketResponse{ StateDiskKey: stateDiskKey, @@ -294,10 +303,13 @@ type dataKeyGetter interface { type certificateAuthority interface { // GetCertificate returns a certificate and private key, signed by the issuer. GetCertificate(certificateRequest []byte) (kubeletCert []byte, err error) + // GetNodeNameFromCSR returns the node name from the CSR. + GetNodeNameFromCSR(csr []byte) (string, error) } type kubeClient interface { GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error + AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string) error AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error } diff --git a/joinservice/internal/server/server_test.go b/joinservice/internal/server/server_test.go index b7da60967..b12d042db 100644 --- a/joinservice/internal/server/server_test.go +++ b/joinservice/internal/server/server_test.go @@ -69,7 +69,7 @@ func TestIssueJoinTicket(t *testing.T) { uuid: testKey, attestation.MeasurementSecretContext: measurementSecret, }}, - ca: stubCA{cert: testCert}, + ca: stubCA{cert: testCert, nodeName: "node"}, kubeClient: stubKubeClient{getComponentsVal: components}, }, "worker node components reference missing": { @@ -78,7 +78,7 @@ func TestIssueJoinTicket(t *testing.T) { uuid: testKey, attestation.MeasurementSecretContext: measurementSecret, }}, - ca: stubCA{cert: testCert}, + ca: stubCA{cert: testCert, nodeName: "node"}, kubeClient: stubKubeClient{getComponentsVal: components}, missingComponentsReferenceFile: true, }, @@ -88,7 +88,7 @@ func TestIssueJoinTicket(t *testing.T) { uuid: testKey, attestation.MeasurementSecretContext: measurementSecret, }}, - ca: stubCA{cert: testCert}, + ca: stubCA{cert: testCert, nodeName: "node"}, kubeClient: stubKubeClient{createConfigMapErr: someErr}, missingComponentsReferenceFile: true, wantErr: true, @@ -99,14 +99,34 @@ func TestIssueJoinTicket(t *testing.T) { uuid: testKey, attestation.MeasurementSecretContext: measurementSecret, }}, - ca: stubCA{cert: testCert}, + ca: stubCA{cert: testCert, nodeName: "node"}, kubeClient: stubKubeClient{getComponentsErr: someErr}, wantErr: true, }, + "Getting Node Name from CSR fails": { + kubeadm: stubTokenGetter{token: testJoinToken}, + kms: stubKeyGetter{dataKeys: map[string][]byte{ + uuid: testKey, + attestation.MeasurementSecretContext: measurementSecret, + }}, + ca: stubCA{cert: testCert, nodeName: "node", getNameErr: someErr}, + kubeClient: stubKubeClient{getComponentsVal: components}, + wantErr: true, + }, + "Cannot add node to JoiningNode CRD": { + kubeadm: stubTokenGetter{token: testJoinToken}, + kms: stubKeyGetter{dataKeys: map[string][]byte{ + uuid: testKey, + attestation.MeasurementSecretContext: measurementSecret, + }}, + ca: stubCA{cert: testCert, nodeName: "node"}, + kubeClient: stubKubeClient{getComponentsVal: components, addNodeToJoiningNodesErr: someErr}, + wantErr: true, + }, "GetDataKey fails": { kubeadm: stubTokenGetter{token: testJoinToken}, kms: stubKeyGetter{dataKeys: make(map[string][]byte), getDataKeyErr: someErr}, - ca: stubCA{cert: testCert}, + ca: stubCA{cert: testCert, nodeName: "node"}, kubeClient: stubKubeClient{getComponentsVal: components}, wantErr: true, }, @@ -116,7 +136,7 @@ func TestIssueJoinTicket(t *testing.T) { uuid: testKey, attestation.MeasurementSecretContext: measurementSecret, }}, - ca: stubCA{cert: testCert}, + ca: stubCA{cert: testCert, nodeName: "node"}, kubeClient: stubKubeClient{getComponentsVal: components}, wantErr: true, }, @@ -126,7 +146,7 @@ func TestIssueJoinTicket(t *testing.T) { uuid: testKey, attestation.MeasurementSecretContext: measurementSecret, }}, - ca: stubCA{getCertErr: someErr}, + ca: stubCA{getCertErr: someErr, nodeName: "node"}, kubeClient: stubKubeClient{getComponentsVal: components}, wantErr: true, }, @@ -140,7 +160,7 @@ func TestIssueJoinTicket(t *testing.T) { uuid: testKey, attestation.MeasurementSecretContext: measurementSecret, }}, - ca: stubCA{cert: testCert}, + ca: stubCA{cert: testCert, nodeName: "node"}, kubeClient: stubKubeClient{getComponentsVal: components}, }, "GetControlPlaneCertificateKey fails": { @@ -150,7 +170,7 @@ func TestIssueJoinTicket(t *testing.T) { uuid: testKey, attestation.MeasurementSecretContext: measurementSecret, }}, - ca: stubCA{cert: testCert}, + ca: stubCA{cert: testCert, nodeName: "node"}, kubeClient: stubKubeClient{getComponentsVal: components}, wantErr: true, }, @@ -177,7 +197,7 @@ func TestIssueJoinTicket(t *testing.T) { ca: tc.ca, joinTokenGetter: tc.kubeadm, dataKeyGetter: tc.kms, - kubeClient: tc.kubeClient, + kubeClient: &tc.kubeClient, log: logger.NewTest(t), } @@ -200,6 +220,8 @@ func TestIssueJoinTicket(t *testing.T) { assert.Equal(tc.kubeadm.token.Token, resp.Token) assert.Equal(tc.ca.cert, resp.KubeletCert) assert.Equal(tc.kubeClient.getComponentsVal.ToJoinProto(), resp.KubernetesComponents) + assert.Equal(tc.ca.nodeName, tc.kubeClient.joiningNodeName) + assert.Equal(tc.kubeClient.getComponentsVal.GetHash(), tc.kubeClient.componentsHash) if tc.isControlPlane { assert.Len(resp.ControlPlaneFiles, len(tc.kubeadm.files)) @@ -288,29 +310,44 @@ func (f stubKeyGetter) GetDataKey(_ context.Context, name string, _ int) ([]byte type stubCA struct { cert []byte getCertErr error + nodeName string + getNameErr error } func (f stubCA) GetCertificate(csr []byte) ([]byte, error) { return f.cert, f.getCertErr } +func (f stubCA) GetNodeNameFromCSR(csr []byte) (string, error) { + return f.nodeName, f.getNameErr +} + type stubKubeClient struct { getComponentsVal versions.ComponentVersions getComponentsErr error createConfigMapErr error - AddReferenceToK8sVersionConfigMapErr error + addReferenceToK8sVersionConfigMapErr error + addNodeToJoiningNodesErr error + joiningNodeName string + componentsHash string } -func (s stubKubeClient) GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error) { +func (s *stubKubeClient) GetComponents(ctx context.Context, configMapName string) (versions.ComponentVersions, error) { return s.getComponentsVal, s.getComponentsErr } -func (s stubKubeClient) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error { +func (s *stubKubeClient) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error { return s.createConfigMapErr } -func (s stubKubeClient) AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error { - return s.AddReferenceToK8sVersionConfigMapErr +func (s *stubKubeClient) AddReferenceToK8sVersionConfigMap(ctx context.Context, k8sVersionsConfigMapName string, componentsConfigMapName string) error { + return s.addReferenceToK8sVersionConfigMapErr +} + +func (s *stubKubeClient) AddNodeToJoiningNodes(ctx context.Context, nodeName string, componentsHash string) error { + s.joiningNodeName = nodeName + s.componentsHash = componentsHash + return s.addNodeToJoiningNodesErr } diff --git a/operators/constellation-node-operator/.gitignore b/operators/constellation-node-operator/.gitignore index c0a7a54ca..f7e60d81d 100644 --- a/operators/constellation-node-operator/.gitignore +++ b/operators/constellation-node-operator/.gitignore @@ -16,6 +16,9 @@ testbin/* # Kubernetes Generated files - skip generated files, except for vendored files +# We hold the charts in the cli/internal/helm directory +chart/ + !vendor/**/zz_generated.* # editor and IDE paraphernalia diff --git a/operators/constellation-node-operator/api/v1alpha1/joiningnodes_types.go b/operators/constellation-node-operator/api/v1alpha1/joiningnodes_types.go new file mode 100644 index 000000000..b5cbaf8af --- /dev/null +++ b/operators/constellation-node-operator/api/v1alpha1/joiningnodes_types.go @@ -0,0 +1,48 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// JoiningNodeSpec defines the components hash which the node should be annotated with. +type JoiningNodeSpec struct { + // Name of the node expected to join. + Name string `json:"name,omitempty"` + // ComponentsHash is the hash of the components that were sent to the node by the join service. + ComponentsHash string `json:"componentshash,omitempty"` +} + +// JoiningNodeStatus defines the observed state of JoiningNode. +type JoiningNodeStatus struct{} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:scope=Cluster + +// JoiningNode is the Schema for the joiningnodes API. +type JoiningNode struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec JoiningNodeSpec `json:"spec,omitempty"` + Status JoiningNodeStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// JoiningNodeList contains a list of JoiningNodes. +type JoiningNodeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []JoiningNode `json:"items"` +} + +func init() { + SchemeBuilder.Register(&JoiningNode{}, &JoiningNodeList{}) +} diff --git a/operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go b/operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go index 3f33a89ee..cb0eb0b9a 100644 --- a/operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -107,6 +107,95 @@ func (in *AutoscalingStrategyStatus) DeepCopy() *AutoscalingStrategyStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JoiningNode) DeepCopyInto(out *JoiningNode) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoiningNode. +func (in *JoiningNode) DeepCopy() *JoiningNode { + if in == nil { + return nil + } + out := new(JoiningNode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *JoiningNode) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JoiningNodeList) DeepCopyInto(out *JoiningNodeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]JoiningNode, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoiningNodeList. +func (in *JoiningNodeList) DeepCopy() *JoiningNodeList { + if in == nil { + return nil + } + out := new(JoiningNodeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *JoiningNodeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JoiningNodeSpec) DeepCopyInto(out *JoiningNodeSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoiningNodeSpec. +func (in *JoiningNodeSpec) DeepCopy() *JoiningNodeSpec { + if in == nil { + return nil + } + out := new(JoiningNodeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JoiningNodeStatus) DeepCopyInto(out *JoiningNodeStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoiningNodeStatus. +func (in *JoiningNodeStatus) DeepCopy() *JoiningNodeStatus { + if in == nil { + return nil + } + out := new(JoiningNodeStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeImage) DeepCopyInto(out *NodeImage) { *out = *in diff --git a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml new file mode 100644 index 000000000..792764195 --- /dev/null +++ b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: joiningnodes.update.edgeless.systems +spec: + group: update.edgeless.systems + names: + kind: JoiningNode + listKind: JoiningNodeList + plural: joiningnodes + singular: joiningnode + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: JoiningNode is the Schema for the joiningnodes API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: JoiningNodeSpec defines the components hash which the node + should be annotated with. + properties: + componentshash: + description: ComponentsHash is the hash of the components that were + sent to the node by the join service. + type: string + name: + description: Name of the node expected to join. + type: string + type: object + status: + description: JoiningNodeStatus defines the observed state of JoiningNode. + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/operators/constellation-node-operator/config/crd/kustomization.yaml b/operators/constellation-node-operator/config/crd/kustomization.yaml index 19b9431f9..09c804a16 100644 --- a/operators/constellation-node-operator/config/crd/kustomization.yaml +++ b/operators/constellation-node-operator/config/crd/kustomization.yaml @@ -3,6 +3,7 @@ # It should be run by config/default resources: - bases/update.edgeless.systems_nodeimages.yaml +- bases/update.edgeless.systems_joiningnodes.yaml - bases/update.edgeless.systems_autoscalingstrategies.yaml - bases/update.edgeless.systems_scalinggroups.yaml - bases/update.edgeless.systems_pendingnodes.yaml @@ -12,6 +13,7 @@ patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_nodeimages.yaml +#- patches/webhook_in_joiningnodes.yaml #- patches/webhook_in_autoscalingstrategies.yaml #- patches/webhook_in_scalinggroups.yaml #- patches/webhook_in_pendingnodes.yaml @@ -20,6 +22,7 @@ patchesStrategicMerge: # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_nodeimages.yaml +#- patches/cainjection_in_joiningnodes.yaml #- patches/cainjection_in_autoscalingstrategies.yaml #- patches/cainjection_in_scalinggroups.yaml #- patches/cainjection_in_pendingnodes.yaml diff --git a/operators/constellation-node-operator/config/crd/patches/cainjection_in_joiningnodes.yaml b/operators/constellation-node-operator/config/crd/patches/cainjection_in_joiningnodes.yaml new file mode 100644 index 000000000..896050d25 --- /dev/null +++ b/operators/constellation-node-operator/config/crd/patches/cainjection_in_joiningnodes.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: joiningnodes.update.edgeless.systems diff --git a/operators/constellation-node-operator/config/crd/patches/webhook_in_joiningnodes.yaml b/operators/constellation-node-operator/config/crd/patches/webhook_in_joiningnodes.yaml new file mode 100644 index 000000000..f544736ad --- /dev/null +++ b/operators/constellation-node-operator/config/crd/patches/webhook_in_joiningnodes.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: joiningnodes.update.edgeless.systems +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/operators/constellation-node-operator/config/rbac/role.yaml b/operators/constellation-node-operator/config/rbac/role.yaml index cee79edec..495920f3f 100644 --- a/operators/constellation-node-operator/config/rbac/role.yaml +++ b/operators/constellation-node-operator/config/rbac/role.yaml @@ -72,6 +72,32 @@ rules: - get - patch - update +- apiGroups: + - update.edgeless.systems + resources: + - joiningnodes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - update.edgeless.systems + resources: + - joiningnodes/finalizers + verbs: + - update +- apiGroups: + - update.edgeless.systems + resources: + - joiningnodes/status + verbs: + - get + - patch + - update - apiGroups: - update.edgeless.systems resources: diff --git a/operators/constellation-node-operator/controllers/joiningnode_controller.go b/operators/constellation-node-operator/controllers/joiningnode_controller.go new file mode 100644 index 000000000..d44966c4d --- /dev/null +++ b/operators/constellation-node-operator/controllers/joiningnode_controller.go @@ -0,0 +1,127 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package controllers + +import ( + "context" + + updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const ( + // NodeKubernetesComponentsHashAnnotationKey is the name of the annotation holding the hash of the installed components of this node. + NodeKubernetesComponentsHashAnnotationKey = "updates.edgeless.systems/kubernetes-components-hash" + + joiningNodeNameKey = ".spec.name" +) + +// JoiningNodesReconciler reconciles a JoiningNode object. +type JoiningNodesReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// NewJoiningNodesReconciler creates a new JoiningNodesReconciler. +func NewJoiningNodesReconciler(client client.Client, scheme *runtime.Scheme) *JoiningNodesReconciler { + return &JoiningNodesReconciler{ + Client: client, + Scheme: scheme, + } +} + +//+kubebuilder:rbac:groups=update.edgeless.systems,resources=joiningnodes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=update.edgeless.systems,resources=joiningnodes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=update.edgeless.systems,resources=joiningnodes/finalizers,verbs=update +//+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch + +// Reconcile annotates the node with the components hash. +func (r *JoiningNodesReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logr := log.FromContext(ctx) + + var joiningNode updatev1alpha1.JoiningNode + if err := r.Get(ctx, req.NamespacedName, &joiningNode); err != nil { + logr.Error(err, "unable to fetch JoiningNodes") + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var node corev1.Node + if err := r.Get(ctx, types.NamespacedName{Name: joiningNode.Spec.Name}, &node); err != nil { + logr.Error(err, "unable to fetch Node") + return err + } + + // add annotations to node + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + node.Annotations[NodeKubernetesComponentsHashAnnotationKey] = joiningNode.Spec.ComponentsHash + return r.Update(ctx, &node) + }) + if err != nil { + logr.Error(err, "unable to update Node") + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := r.Delete(ctx, &joiningNode); err != nil { + logr.Error(err, "unable to delete JoiningNode") + return err + } + return nil + }) + if err != nil { + logr.Error(err, "unable to delete JoiningNode") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *JoiningNodesReconciler) SetupWithManager(mgr ctrl.Manager) error { + // index joining nodes by nodename. + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &updatev1alpha1.JoiningNode{}, joiningNodeNameKey, func(rawObj client.Object) []string { + joiningNode := rawObj.(*updatev1alpha1.JoiningNode) + return []string{joiningNode.Spec.Name} + }); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&updatev1alpha1.JoiningNode{}). + Watches( + &source.Kind{Type: &corev1.Node{}}, + handler.EnqueueRequestsFromMapFunc(r.findAllJoiningNodes), + ). + Complete(r) +} + +func (r *JoiningNodesReconciler) findAllJoiningNodes(obj client.Object) []reconcile.Request { + var joiningNodesList updatev1alpha1.JoiningNodeList + err := r.List(context.TODO(), &joiningNodesList, client.MatchingFields{joiningNodeNameKey: obj.GetName()}) + if err != nil { + return []reconcile.Request{} + } + requests := make([]reconcile.Request, len(joiningNodesList.Items)) + for i, item := range joiningNodesList.Items { + requests[i] = reconcile.Request{ + NamespacedName: types.NamespacedName{Name: item.GetName()}, + } + } + return requests +} diff --git a/operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go b/operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go new file mode 100644 index 000000000..00f125e1d --- /dev/null +++ b/operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go @@ -0,0 +1,143 @@ +//go:build integration + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package controllers + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1" +) + +var _ = Describe("JoiningNode controller", func() { + const ( + nodeName1 = "node-name-1" + nodeName2 = "node-name-2" + componentsHash1 = "test-hash-1" + componentsHash2 = "test-hash-2" + + timeout = time.Second * 20 + duration = time.Second * 2 + interval = time.Millisecond * 250 + ) + Context("When changing a joining node resource spec", func() { + It("Should annotate the corresponding node when creating the CRD first", func() { + By("creating a joining node resource") + ctx := context.Background() + joiningNode := &updatev1alpha1.JoiningNode{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "update.edgeless.systems/v1alpha1", + Kind: "JoiningNode", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName1, + }, + Spec: updatev1alpha1.JoiningNodeSpec{ + Name: nodeName1, + ComponentsHash: componentsHash1, + }, + } + Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed()) + createdJoiningNode := &updatev1alpha1.JoiningNode{} + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdJoiningNode) + }, timeout, interval).Should(Succeed()) + Expect(createdJoiningNode.Spec.Name).Should(Equal(nodeName1)) + Expect(createdJoiningNode.Spec.ComponentsHash).Should(Equal(componentsHash1)) + + By("creating a node") + node := &corev1.Node{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "update.edgeless.systems/v1alpha1", + Kind: "Node", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName1, + }, + Spec: corev1.NodeSpec{}, + } + Expect(k8sClient.Create(ctx, node)).Should(Succeed()) + createdNode := &corev1.Node{} + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdNode) + }, timeout, interval).Should(Succeed()) + Expect(createdNode.ObjectMeta.Name).Should(Equal(nodeName1)) + + By("annotating the node") + Eventually(func() string { + _ = k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdNode) + return createdNode.Annotations[NodeKubernetesComponentsHashAnnotationKey] + }, timeout, interval).Should(Equal(componentsHash1)) + + By("deleting the joining node resource") + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode) + }, timeout, interval).ShouldNot(Succeed()) + }) + }) + It("Should annotate the corresponding node when creating the node first", func() { + ctx := context.Background() + By("creating a node") + node := &corev1.Node{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "update.edgeless.systems/v1alpha1", + Kind: "Node", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName2, + }, + Spec: corev1.NodeSpec{}, + } + Expect(k8sClient.Create(ctx, node)).Should(Succeed()) + createdNode := &corev1.Node{} + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName2}, createdNode) + }, timeout, interval).Should(Succeed()) + Expect(createdNode.ObjectMeta.Name).Should(Equal(nodeName2)) + + By("creating a joining node resource") + joiningNode := &updatev1alpha1.JoiningNode{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "update.edgeless.systems/v1alpha1", + Kind: "JoiningNode", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName2, + }, + Spec: updatev1alpha1.JoiningNodeSpec{ + Name: nodeName2, + ComponentsHash: componentsHash2, + }, + } + Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed()) + createdJoiningNode := &updatev1alpha1.JoiningNode{} + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode) + }, timeout, interval).Should(Succeed()) + Expect(createdJoiningNode.Spec.Name).Should(Equal(nodeName2)) + Expect(createdJoiningNode.Spec.ComponentsHash).Should(Equal(componentsHash2)) + + By("annotating the node") + Eventually(func() string { + _ = k8sClient.Get(ctx, types.NamespacedName{Name: createdNode.Name}, createdNode) + return createdNode.Annotations[NodeKubernetesComponentsHashAnnotationKey] + }, timeout, interval).Should(Equal(componentsHash2)) + + By("deleting the joining node resource") + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode) + }, timeout, interval).ShouldNot(Succeed()) + }) +}) diff --git a/operators/constellation-node-operator/controllers/suite_test.go b/operators/constellation-node-operator/controllers/suite_test.go index 7cc467122..b24d506a0 100644 --- a/operators/constellation-node-operator/controllers/suite_test.go +++ b/operators/constellation-node-operator/controllers/suite_test.go @@ -93,6 +93,12 @@ var _ = BeforeSuite(func() { }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) + err = (&JoiningNodesReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + err = (&ScalingGroupReconciler{ scalingGroupUpdater: fakes.scalingGroupUpdater, Client: k8sManager.GetClient(), diff --git a/operators/constellation-node-operator/main.go b/operators/constellation-node-operator/main.go index 7a9fc4b6e..b2244e15e 100644 --- a/operators/constellation-node-operator/main.go +++ b/operators/constellation-node-operator/main.go @@ -145,6 +145,13 @@ func main() { setupLog.Error(err, "Unable to create controller", "controller", "AutoscalingStrategy") os.Exit(1) } + if err = (&controllers.JoiningNodesReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "Unable to create controller", "controller", "JoiningNode") + os.Exit(1) + } if err = controllers.NewScalingGroupReconciler( cspClient, mgr.GetClient(), mgr.GetScheme(), ).SetupWithManager(mgr); err != nil {