AB#2234: Introduce AddNodeSelectorsToDeployment

Add the above function to the different client interfaces.
Remove previously used Command.Exec call.

bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go

@@ -120,6 +120,30 @@ func (c *Client) AddTolerationsToDeployment(ctx context.Context, tolerations []c
 	if err != nil {
 		return err
 	}
-
+	return nil
+}
+
+func (c *Client) AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string) error {
+	deployments := c.clientset.AppsV1().Deployments(corev1.NamespaceAll)
+
+	// retry resource update if an error occurs
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		result, err := deployments.Get(ctx, name, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("Failed to get latest version of Deployment: %v", err)
+		}
+
+		for k, v := range selectors {
+			result.Spec.Template.Spec.NodeSelector[k] = v
+		}
+
+		if _, err = deployments.Update(ctx, result, metav1.UpdateOptions{}); err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
 	return nil
 }

bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go

@@ -75,6 +75,23 @@ var (
 			},
 		},
 	}
+	tolerationsDeployment = appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-deployment",
+		},
+	}
+	selectorsDeployment = appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-deployment",
+		},
+		Spec: appsv1.DeploymentSpec{
+			Template: k8s.PodTemplateSpec{
+				Spec: k8s.PodSpec{
+					NodeSelector: map[string]string{},
+				},
+			},
+		},
+	}
 	nginxDeplJSON, _ = marshalJSON(nginxDeployment)
 	nginxDeplYAML, _ = marshalYAML(nginxDeployment)
 )
@@ -284,28 +301,15 @@ func TestGetObjects(t *testing.T) {
 func TestAddTolerationsToDeployment(t *testing.T) {
 	testCases := map[string]struct {
 		name        string
-		deployment  appsv1.Deployment
 		tolerations []corev1.Toleration
 		wantErr     bool
 	}{
 		"Success": {
 			name: "test-deployment",
-			deployment: appsv1.Deployment{
-				ObjectMeta: metav1.ObjectMeta{
-					Name: "test-deployment",
-				},
-			},
-			tolerations: []corev1.Toleration{},
 		},
 		"Specifying non-existent deployment fails": {
-			name: "wrong-name",
+			name:    "wrong-name",
-			deployment: appsv1.Deployment{
+			wantErr: true,
-				ObjectMeta: metav1.ObjectMeta{
-					Name: "test-deployment",
-				},
-			},
-			tolerations: []corev1.Toleration{},
-			wantErr:     true,
 		},
 	}
 
@@ -314,7 +318,7 @@ func TestAddTolerationsToDeployment(t *testing.T) {
 			assert := assert.New(t)
 			require := require.New(t)
 
-			client := newClientWithFakes(t, map[string]string{}, &tc.deployment)
+			client := newClientWithFakes(t, map[string]string{}, &tolerationsDeployment)
 			err := client.AddTolerationsToDeployment(context.Background(), tc.tolerations, tc.name)
 			if tc.wantErr {
 				assert.Error(err)
@@ -324,3 +328,35 @@ func TestAddTolerationsToDeployment(t *testing.T) {
 		})
 	}
 }
+
+func TestAddNodeSelectorsToDeployment(t *testing.T) {
+	testCases := map[string]struct {
+		name      string
+		selectors map[string]string
+		wantErr   bool
+	}{
+		"Success": {
+			name:      "test-deployment",
+			selectors: map[string]string{"some-key": "some-value"},
+		},
+		"Specifying non-existent deployment fails": {
+			name:    "wrong-name",
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			client := newClientWithFakes(t, map[string]string{}, &selectorsDeployment)
+			err := client.AddNodeSelectorsToDeployment(context.Background(), tc.selectors, tc.name)
+			if tc.wantErr {
+				assert.Error(err)
+				return
+			}
+			require.NoError(err)
+		})
+	}
+}

bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go

@@ -21,6 +21,7 @@ type Client interface {
 	GetObjects(resources resources.Marshaler) ([]*resource.Info, error)
 	CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
 	AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error
+	AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string) error
 }
 
 // clientGenerator can generate new clients from a kubeconfig.
@@ -98,3 +99,15 @@ func (k *Kubectl) AddTolerationsToDeployment(ctx context.Context, tolerations []
 	return nil
 }
 
+func (k *Kubectl) AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string) error {
+	client, err := k.clientGenerator.NewClient(k.kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	if err = client.AddNodeSelectorsToDeployment(ctx, selectors, name); err != nil {
+		return err
+	}
+
+	return nil
+}

bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go

@@ -22,6 +22,7 @@ type stubClient struct {
 	getObjectsErr                  error
 	createConfigMapErr             error
 	addTolerationsToDeploymentErr  error
+	addNodeSelectorToDeploymentErr error
 }
 
 func (s *stubClient) ApplyOneObject(info *resource.Info, forceConflicts bool) error {
@@ -40,6 +41,10 @@ func (s *stubClient) AddTolerationsToDeployment(ctx context.Context, tolerations
 	return s.addTolerationsToDeploymentErr
 }
 
+func (s *stubClient) AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string) error {
+	return s.addNodeSelectorToDeploymentErr
+}
+
 type stubClientGenerator struct {
 	applyOneObjectErr              error
 	getObjectsInfos                []*resource.Info
@@ -47,6 +52,7 @@ type stubClientGenerator struct {
 	newClientErr                   error
 	createConfigMapErr             error
 	addTolerationsToDeploymentErr  error
+	addNodeSelectorToDeploymentErr error
 }
 
 func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) {
@@ -56,6 +62,7 @@ func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) {
 		s.getObjectsErr,
 		s.createConfigMapErr,
 		s.addTolerationsToDeploymentErr,
+		s.addNodeSelectorToDeploymentErr,
 	}, s.newClientErr
 }
 

bootstrapper/internal/kubernetes/k8sapi/util.go

@@ -46,6 +46,7 @@ type Client interface {
 	SetKubeconfig(kubeconfig []byte)
 	CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
 	AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error
+	AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string) error
 }
 
 type installer interface {
@@ -227,10 +228,6 @@ func (k *KubernetesUtil) setupGCPPodNetwork(ctx context.Context, nodeName, nodeP
 	}
 
 	// allow coredns to run on uninitialized nodes (required by cloud-controller-manager)
-	err = exec.CommandContext(ctx, kubectlPath, "--kubeconfig", kubeConfig, "-n", "kube-system", "patch", "deployment", "coredns", "--type", "json", "-p", "[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.cloudprovider.kubernetes.io/uninitialized\",\"value\":\"true\",\"effect\":\"NoSchedule\"}},{\"op\":\"add\",\"path\":\"/spec/template/spec/nodeSelector\",\"value\":{\"node-role.kubernetes.io/control-plane\":\"\"}}]").Run()
-	if err != nil {
-		return err
-	}
 	tolerations := []corev1.Toleration{
 		{
 			Key:    "node.cloudprovider.kubernetes.io/uninitialized",
@@ -241,6 +238,12 @@ func (k *KubernetesUtil) setupGCPPodNetwork(ctx context.Context, nodeName, nodeP
 	if err = kubectl.AddTolerationsToDeployment(ctx, tolerations, "coredns"); err != nil {
 		return err
 	}
+	selectors := map[string]string{
+		"node-role.kubernetes.io/control-plane": "",
+	}
+	if err = kubectl.AddNodeSelectorsToDeployment(ctx, selectors, "coredns"); err != nil {
+		return err
+	}
 
 	ciliumInstall := exec.CommandContext(ctx, "cilium", "install", "--ipam", "kubernetes", "--ipv4-native-routing-cidr", subnetworkPodCIDR,
 		"--helm-set", "endpointRoutes.enabled=true,tunnel=disabled,encryption.enabled=true,encryption.type=wireguard,l7Proxy=false")

bootstrapper/internal/kubernetes/kubernetes_test.go

@@ -606,6 +606,7 @@ type stubKubectl struct {
 	ApplyErr                         error
 	createConfigMapErr               error
 	AddTolerationsToDeploymentErr    error
+	AddTNodeSelectorsToDeploymentErr error
 
 	resources   []resources.Marshaler
 	kubeconfigs [][]byte
@@ -628,6 +629,10 @@ func (s *stubKubectl) AddTolerationsToDeployment(ctx context.Context, toleration
 	return s.AddTolerationsToDeploymentErr
 }
 
+func (s *stubKubectl) AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string) error {
+	return s.AddTNodeSelectorsToDeploymentErr
+}
+
 type stubKubeconfigReader struct {
 	Kubeconfig []byte
 	ReadErr    error