From 0618a000a7d71bcfc198eaed4b4c201aa4de6fff Mon Sep 17 00:00:00 2001 From: Malte Poll Date: Mon, 18 Jul 2022 16:39:48 +0200 Subject: [PATCH] [node operator] nodeimage controller: remove control-plane nodes from etcd cluster before deleting k8s node object Signed-off-by: Malte Poll --- .../controllers/nodeimage_controller.go | 22 +++++++++++++++++-- operators/constellation-node-operator/main.go | 16 +++++++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/operators/constellation-node-operator/controllers/nodeimage_controller.go b/operators/constellation-node-operator/controllers/nodeimage_controller.go index 26bd532ec..f87edfcde 100644 --- a/operators/constellation-node-operator/controllers/nodeimage_controller.go +++ b/operators/constellation-node-operator/controllers/nodeimage_controller.go @@ -28,7 +28,7 @@ import ( const ( // nodeOverprovisionLimit is the maximum number of extra nodes created during the update procedure at any point in time. - nodeOverprovisionLimit = 2 + nodeOverprovisionLimit = 4 // nodeJoinTimeout is the time limit pending nodes have to join the cluster before being terminated. nodeJoinTimeout = time.Minute * 15 // nodeLeaveTimeout is the time limit pending nodes have to leave the cluster and being terminated. @@ -47,14 +47,16 @@ const ( // NodeImageReconciler reconciles a NodeImage object type NodeImageReconciler struct { nodeReplacer + etcdRemover client.Client Scheme *runtime.Scheme } // NewNodeImageReconciler creates a new NodeImageReconciler. -func NewNodeImageReconciler(nodeReplacer nodeReplacer, client client.Client, scheme *runtime.Scheme) *NodeImageReconciler { +func NewNodeImageReconciler(nodeReplacer nodeReplacer, etcdRemover etcdRemover, client client.Client, scheme *runtime.Scheme) *NodeImageReconciler { return &NodeImageReconciler{ nodeReplacer: nodeReplacer, + etcdRemover: etcdRemover, Client: client, Scheme: scheme, } @@ -441,6 +443,17 @@ func (r *NodeImageReconciler) deleteNode(ctx context.Context, controller metav1. } // node is unused & ready to be replaced + if nodeutil.IsControlPlaneNode(&node) { + nodeVPCIP, err := nodeutil.VPCIP(&node) + if err != nil { + logr.Error(err, "Unable to get node VPC IP") + return false, err + } + if err := r.RemoveEtcdMemberFromCluster(ctx, nodeVPCIP); err != nil { + logr.Error(err, "Unable to remove etcd member from cluster") + return false, err + } + } if err := r.Delete(ctx, &node); err != nil { logr.Error(err, "Deleting node") return false, err @@ -778,3 +791,8 @@ type nodeReplacer interface { // DeleteNode starts the termination of the node at the CSP. DeleteNode(ctx context.Context, providerID string) error } + +type etcdRemover interface { + // RemoveEtcdMemberFromCluster removes an etcd member from the cluster. + RemoveEtcdMemberFromCluster(ctx context.Context, vpcIP string) error +} diff --git a/operators/constellation-node-operator/main.go b/operators/constellation-node-operator/main.go index 7375159a9..9b3020028 100644 --- a/operators/constellation-node-operator/main.go +++ b/operators/constellation-node-operator/main.go @@ -15,11 +15,13 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/api/v1alpha1" "github.com/edgelesssys/constellation/operators/constellation-node-operator/controllers" + "github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/etcd" nodemaintenancev1beta1 "github.com/medik8s/node-maintenance-operator/api/v1beta1" //+kubebuilder:scaffold:imports ) @@ -79,8 +81,20 @@ func main() { os.Exit(1) } + k8sClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme}) + if err != nil { + setupLog.Error(err, "Unable to create k8s client") + os.Exit(1) + } + etcdClient, err := etcd.New(k8sClient) + if err != nil { + setupLog.Error(err, "Unable to create etcd client") + os.Exit(1) + } + defer etcdClient.Close() + if err = controllers.NewNodeImageReconciler( - cspClient, mgr.GetClient(), mgr.GetScheme(), + cspClient, etcdClient, mgr.GetClient(), mgr.GetScheme(), ).SetupWithManager(mgr); err != nil { setupLog.Error(err, "Unable to create controller", "controller", "NodeImage") os.Exit(1)