[node operator] nodeimage controller: remove control-plane nodes from etcd cluster before deleting k8s node object

Signed-off-by: Malte Poll <mp@edgeless.systems>

operators/constellation-node-operator/controllers/nodeimage_controller.go

@@ -28,7 +28,7 @@ import (
 
 const (
 	// nodeOverprovisionLimit is the maximum number of extra nodes created during the update procedure at any point in time.
-	nodeOverprovisionLimit = 2
+	nodeOverprovisionLimit = 4
 	// nodeJoinTimeout is the time limit pending nodes have to join the cluster before being terminated.
 	nodeJoinTimeout = time.Minute * 15
 	// nodeLeaveTimeout is the time limit pending nodes have to leave the cluster and being terminated.
@@ -47,14 +47,16 @@ const (
 // NodeImageReconciler reconciles a NodeImage object
 type NodeImageReconciler struct {
 	nodeReplacer
+	etcdRemover
 	client.Client
 	Scheme *runtime.Scheme
 }
 
 // NewNodeImageReconciler creates a new NodeImageReconciler.
-func NewNodeImageReconciler(nodeReplacer nodeReplacer, client client.Client, scheme *runtime.Scheme) *NodeImageReconciler {
+func NewNodeImageReconciler(nodeReplacer nodeReplacer, etcdRemover etcdRemover, client client.Client, scheme *runtime.Scheme) *NodeImageReconciler {
 	return &NodeImageReconciler{
 		nodeReplacer: nodeReplacer,
+		etcdRemover:  etcdRemover,
 		Client:       client,
 		Scheme:       scheme,
 	}
@@ -441,6 +443,17 @@ func (r *NodeImageReconciler) deleteNode(ctx context.Context, controller metav1.
 	}
 
 	// node is unused & ready to be replaced
+	if nodeutil.IsControlPlaneNode(&node) {
+		nodeVPCIP, err := nodeutil.VPCIP(&node)
+		if err != nil {
+			logr.Error(err, "Unable to get node VPC IP")
+			return false, err
+		}
+		if err := r.RemoveEtcdMemberFromCluster(ctx, nodeVPCIP); err != nil {
+			logr.Error(err, "Unable to remove etcd member from cluster")
+			return false, err
+		}
+	}
 	if err := r.Delete(ctx, &node); err != nil {
 		logr.Error(err, "Deleting node")
 		return false, err
@@ -778,3 +791,8 @@ type nodeReplacer interface {
 	// DeleteNode starts the termination of the node at the CSP.
 	DeleteNode(ctx context.Context, providerID string) error
 }
+
+type etcdRemover interface {
+	// RemoveEtcdMemberFromCluster removes an etcd member from the cluster.
+	RemoveEtcdMemberFromCluster(ctx context.Context, vpcIP string) error
+}

operators/constellation-node-operator/main.go

@@ -15,11 +15,13 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/api/v1alpha1"
 	"github.com/edgelesssys/constellation/operators/constellation-node-operator/controllers"
+	"github.com/edgelesssys/constellation/operators/constellation-node-operator/internal/etcd"
 	nodemaintenancev1beta1 "github.com/medik8s/node-maintenance-operator/api/v1beta1"
 	//+kubebuilder:scaffold:imports
 )
@@ -79,8 +81,20 @@ func main() {
 		os.Exit(1)
 	}
 
+	k8sClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
+	if err != nil {
+		setupLog.Error(err, "Unable to create k8s client")
+		os.Exit(1)
+	}
+	etcdClient, err := etcd.New(k8sClient)
+	if err != nil {
+		setupLog.Error(err, "Unable to create etcd client")
+		os.Exit(1)
+	}
+	defer etcdClient.Close()
+
 	if err = controllers.NewNodeImageReconciler(
-		cspClient, mgr.GetClient(), mgr.GetScheme(),
+		cspClient, etcdClient, mgr.GetClient(), mgr.GetScheme(),
 	).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "Unable to create controller", "controller", "NodeImage")
 		os.Exit(1)