mirror of
https://github.com/edgelesssys/constellation.git
synced 2025-05-02 06:16:08 -04:00
k8supdates: label nodes with k8s component hash
This commit is contained in:
Leonard Cohnen
3u13r
parent
1466c12972
commit
a1161ae05d
30 changed files with 869 additions and 18 deletions
|
|
@ -0,0 +1,127 @@
|
|||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/util/retry"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/handler"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
"sigs.k8s.io/controller-runtime/pkg/source"
|
||||
)
|
||||
|
||||
const (
|
||||
// NodeKubernetesComponentsHashAnnotationKey is the name of the annotation holding the hash of the installed components of this node.
|
||||
NodeKubernetesComponentsHashAnnotationKey = "updates.edgeless.systems/kubernetes-components-hash"
|
||||
|
||||
joiningNodeNameKey = ".spec.name"
|
||||
)
|
||||
|
||||
// JoiningNodesReconciler reconciles a JoiningNode object.
|
||||
type JoiningNodesReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
}
|
||||
|
||||
// NewJoiningNodesReconciler creates a new JoiningNodesReconciler.
|
||||
func NewJoiningNodesReconciler(client client.Client, scheme *runtime.Scheme) *JoiningNodesReconciler {
|
||||
return &JoiningNodesReconciler{
|
||||
Client: client,
|
||||
Scheme: scheme,
|
||||
}
|
||||
}
|
||||
|
||||
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=joiningnodes,verbs=get;list;watch;create;update;patch;delete
|
||||
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=joiningnodes/status,verbs=get;update;patch
|
||||
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=joiningnodes/finalizers,verbs=update
|
||||
//+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch
|
||||
|
||||
// Reconcile annotates the node with the components hash.
|
||||
func (r *JoiningNodesReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
logr := log.FromContext(ctx)
|
||||
|
||||
var joiningNode updatev1alpha1.JoiningNode
|
||||
if err := r.Get(ctx, req.NamespacedName, &joiningNode); err != nil {
|
||||
logr.Error(err, "unable to fetch JoiningNodes")
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
|
||||
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
var node corev1.Node
|
||||
if err := r.Get(ctx, types.NamespacedName{Name: joiningNode.Spec.Name}, &node); err != nil {
|
||||
logr.Error(err, "unable to fetch Node")
|
||||
return err
|
||||
}
|
||||
|
||||
// add annotations to node
|
||||
if node.Annotations == nil {
|
||||
node.Annotations = map[string]string{}
|
||||
}
|
||||
node.Annotations[NodeKubernetesComponentsHashAnnotationKey] = joiningNode.Spec.ComponentsHash
|
||||
return r.Update(ctx, &node)
|
||||
})
|
||||
if err != nil {
|
||||
logr.Error(err, "unable to update Node")
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
|
||||
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
if err := r.Delete(ctx, &joiningNode); err != nil {
|
||||
logr.Error(err, "unable to delete JoiningNode")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
logr.Error(err, "unable to delete JoiningNode")
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
// SetupWithManager sets up the controller with the Manager.
|
||||
func (r *JoiningNodesReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
// index joining nodes by nodename.
|
||||
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &updatev1alpha1.JoiningNode{}, joiningNodeNameKey, func(rawObj client.Object) []string {
|
||||
joiningNode := rawObj.(*updatev1alpha1.JoiningNode)
|
||||
return []string{joiningNode.Spec.Name}
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&updatev1alpha1.JoiningNode{}).
|
||||
Watches(
|
||||
&source.Kind{Type: &corev1.Node{}},
|
||||
handler.EnqueueRequestsFromMapFunc(r.findAllJoiningNodes),
|
||||
).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
func (r *JoiningNodesReconciler) findAllJoiningNodes(obj client.Object) []reconcile.Request {
|
||||
var joiningNodesList updatev1alpha1.JoiningNodeList
|
||||
err := r.List(context.TODO(), &joiningNodesList, client.MatchingFields{joiningNodeNameKey: obj.GetName()})
|
||||
if err != nil {
|
||||
return []reconcile.Request{}
|
||||
}
|
||||
requests := make([]reconcile.Request, len(joiningNodesList.Items))
|
||||
for i, item := range joiningNodesList.Items {
|
||||
requests[i] = reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: item.GetName()},
|
||||
}
|
||||
}
|
||||
return requests
|
||||
}
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
//go:build integration
|
||||
|
||||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
|
||||
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("JoiningNode controller", func() {
|
||||
const (
|
||||
nodeName1 = "node-name-1"
|
||||
nodeName2 = "node-name-2"
|
||||
componentsHash1 = "test-hash-1"
|
||||
componentsHash2 = "test-hash-2"
|
||||
|
||||
timeout = time.Second * 20
|
||||
duration = time.Second * 2
|
||||
interval = time.Millisecond * 250
|
||||
)
|
||||
Context("When changing a joining node resource spec", func() {
|
||||
It("Should annotate the corresponding node when creating the CRD first", func() {
|
||||
By("creating a joining node resource")
|
||||
ctx := context.Background()
|
||||
joiningNode := &updatev1alpha1.JoiningNode{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "update.edgeless.systems/v1alpha1",
|
||||
Kind: "JoiningNode",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: nodeName1,
|
||||
},
|
||||
Spec: updatev1alpha1.JoiningNodeSpec{
|
||||
Name: nodeName1,
|
||||
ComponentsHash: componentsHash1,
|
||||
},
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed())
|
||||
createdJoiningNode := &updatev1alpha1.JoiningNode{}
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdJoiningNode)
|
||||
}, timeout, interval).Should(Succeed())
|
||||
Expect(createdJoiningNode.Spec.Name).Should(Equal(nodeName1))
|
||||
Expect(createdJoiningNode.Spec.ComponentsHash).Should(Equal(componentsHash1))
|
||||
|
||||
By("creating a node")
|
||||
node := &corev1.Node{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "update.edgeless.systems/v1alpha1",
|
||||
Kind: "Node",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: nodeName1,
|
||||
},
|
||||
Spec: corev1.NodeSpec{},
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, node)).Should(Succeed())
|
||||
createdNode := &corev1.Node{}
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdNode)
|
||||
}, timeout, interval).Should(Succeed())
|
||||
Expect(createdNode.ObjectMeta.Name).Should(Equal(nodeName1))
|
||||
|
||||
By("annotating the node")
|
||||
Eventually(func() string {
|
||||
_ = k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdNode)
|
||||
return createdNode.Annotations[NodeKubernetesComponentsHashAnnotationKey]
|
||||
}, timeout, interval).Should(Equal(componentsHash1))
|
||||
|
||||
By("deleting the joining node resource")
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode)
|
||||
}, timeout, interval).ShouldNot(Succeed())
|
||||
})
|
||||
})
|
||||
It("Should annotate the corresponding node when creating the node first", func() {
|
||||
ctx := context.Background()
|
||||
By("creating a node")
|
||||
node := &corev1.Node{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "update.edgeless.systems/v1alpha1",
|
||||
Kind: "Node",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: nodeName2,
|
||||
},
|
||||
Spec: corev1.NodeSpec{},
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, node)).Should(Succeed())
|
||||
createdNode := &corev1.Node{}
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName2}, createdNode)
|
||||
}, timeout, interval).Should(Succeed())
|
||||
Expect(createdNode.ObjectMeta.Name).Should(Equal(nodeName2))
|
||||
|
||||
By("creating a joining node resource")
|
||||
joiningNode := &updatev1alpha1.JoiningNode{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "update.edgeless.systems/v1alpha1",
|
||||
Kind: "JoiningNode",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: nodeName2,
|
||||
},
|
||||
Spec: updatev1alpha1.JoiningNodeSpec{
|
||||
Name: nodeName2,
|
||||
ComponentsHash: componentsHash2,
|
||||
},
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed())
|
||||
createdJoiningNode := &updatev1alpha1.JoiningNode{}
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode)
|
||||
}, timeout, interval).Should(Succeed())
|
||||
Expect(createdJoiningNode.Spec.Name).Should(Equal(nodeName2))
|
||||
Expect(createdJoiningNode.Spec.ComponentsHash).Should(Equal(componentsHash2))
|
||||
|
||||
By("annotating the node")
|
||||
Eventually(func() string {
|
||||
_ = k8sClient.Get(ctx, types.NamespacedName{Name: createdNode.Name}, createdNode)
|
||||
return createdNode.Annotations[NodeKubernetesComponentsHashAnnotationKey]
|
||||
}, timeout, interval).Should(Equal(componentsHash2))
|
||||
|
||||
By("deleting the joining node resource")
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode)
|
||||
}, timeout, interval).ShouldNot(Succeed())
|
||||
})
|
||||
})
|
||||
|
|
@ -93,6 +93,12 @@ var _ = BeforeSuite(func() {
|
|||
}).SetupWithManager(k8sManager)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = (&JoiningNodesReconciler{
|
||||
Client: k8sManager.GetClient(),
|
||||
Scheme: k8sManager.GetScheme(),
|
||||
}).SetupWithManager(k8sManager)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = (&ScalingGroupReconciler{
|
||||
scalingGroupUpdater: fakes.scalingGroupUpdater,
|
||||
Client: k8sManager.GetClient(),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue