diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go b/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go index b00e89749..ab3bb4657 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go @@ -120,6 +120,30 @@ func (c *Client) AddTolerationsToDeployment(ctx context.Context, tolerations []c if err != nil { return err } - + return nil +} + +func (c *Client) AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string) error { + deployments := c.clientset.AppsV1().Deployments(corev1.NamespaceAll) + + // retry resource update if an error occurs + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + result, err := deployments.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("Failed to get latest version of Deployment: %v", err) + } + + for k, v := range selectors { + result.Spec.Template.Spec.NodeSelector[k] = v + } + + if _, err = deployments.Update(ctx, result, metav1.UpdateOptions{}); err != nil { + return err + } + return nil + }) + if err != nil { + return err + } return nil } diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go b/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go index 32a3c5462..8189dd78e 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go @@ -75,6 +75,23 @@ var ( }, }, } + tolerationsDeployment = appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + } + selectorsDeployment = appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + Spec: appsv1.DeploymentSpec{ + Template: k8s.PodTemplateSpec{ + Spec: k8s.PodSpec{ + NodeSelector: map[string]string{}, + }, + }, + }, + } nginxDeplJSON, _ = marshalJSON(nginxDeployment) nginxDeplYAML, _ = marshalYAML(nginxDeployment) ) @@ -284,28 +301,15 @@ func TestGetObjects(t *testing.T) { func TestAddTolerationsToDeployment(t *testing.T) { testCases := map[string]struct { name string - deployment appsv1.Deployment tolerations []corev1.Toleration wantErr bool }{ "Success": { name: "test-deployment", - deployment: appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-deployment", - }, - }, - tolerations: []corev1.Toleration{}, }, "Specifying non-existent deployment fails": { - name: "wrong-name", - deployment: appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-deployment", - }, - }, - tolerations: []corev1.Toleration{}, - wantErr: true, + name: "wrong-name", + wantErr: true, }, } @@ -314,7 +318,7 @@ func TestAddTolerationsToDeployment(t *testing.T) { assert := assert.New(t) require := require.New(t) - client := newClientWithFakes(t, map[string]string{}, &tc.deployment) + client := newClientWithFakes(t, map[string]string{}, &tolerationsDeployment) err := client.AddTolerationsToDeployment(context.Background(), tc.tolerations, tc.name) if tc.wantErr { assert.Error(err) @@ -324,3 +328,35 @@ func TestAddTolerationsToDeployment(t *testing.T) { }) } } + +func TestAddNodeSelectorsToDeployment(t *testing.T) { + testCases := map[string]struct { + name string + selectors map[string]string + wantErr bool + }{ + "Success": { + name: "test-deployment", + selectors: map[string]string{"some-key": "some-value"}, + }, + "Specifying non-existent deployment fails": { + name: "wrong-name", + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := newClientWithFakes(t, map[string]string{}, &selectorsDeployment) + err := client.AddNodeSelectorsToDeployment(context.Background(), tc.selectors, tc.name) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + }) + } +} diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go b/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go index 18983cd9d..2eda3152e 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go @@ -21,6 +21,7 @@ type Client interface { GetObjects(resources resources.Marshaler) ([]*resource.Info, error) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error + AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string) error } // clientGenerator can generate new clients from a kubeconfig. @@ -98,3 +99,15 @@ func (k *Kubectl) AddTolerationsToDeployment(ctx context.Context, tolerations [] return nil } +func (k *Kubectl) AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string) error { + client, err := k.clientGenerator.NewClient(k.kubeconfig) + if err != nil { + return err + } + + if err = client.AddNodeSelectorsToDeployment(ctx, selectors, name); err != nil { + return err + } + + return nil +} diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go b/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go index 5357586b8..96563ee33 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go @@ -22,6 +22,7 @@ type stubClient struct { getObjectsErr error createConfigMapErr error addTolerationsToDeploymentErr error + addNodeSelectorToDeploymentErr error } func (s *stubClient) ApplyOneObject(info *resource.Info, forceConflicts bool) error { @@ -40,6 +41,10 @@ func (s *stubClient) AddTolerationsToDeployment(ctx context.Context, tolerations return s.addTolerationsToDeploymentErr } +func (s *stubClient) AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string) error { + return s.addNodeSelectorToDeploymentErr +} + type stubClientGenerator struct { applyOneObjectErr error getObjectsInfos []*resource.Info @@ -47,6 +52,7 @@ type stubClientGenerator struct { newClientErr error createConfigMapErr error addTolerationsToDeploymentErr error + addNodeSelectorToDeploymentErr error } func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) { @@ -56,6 +62,7 @@ func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) { s.getObjectsErr, s.createConfigMapErr, s.addTolerationsToDeploymentErr, + s.addNodeSelectorToDeploymentErr, }, s.newClientErr } diff --git a/bootstrapper/internal/kubernetes/k8sapi/util.go b/bootstrapper/internal/kubernetes/k8sapi/util.go index 4fcaf44b0..12066138f 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/util.go +++ b/bootstrapper/internal/kubernetes/k8sapi/util.go @@ -46,6 +46,7 @@ type Client interface { SetKubeconfig(kubeconfig []byte) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error + AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string) error } type installer interface { @@ -227,10 +228,6 @@ func (k *KubernetesUtil) setupGCPPodNetwork(ctx context.Context, nodeName, nodeP } // allow coredns to run on uninitialized nodes (required by cloud-controller-manager) - err = exec.CommandContext(ctx, kubectlPath, "--kubeconfig", kubeConfig, "-n", "kube-system", "patch", "deployment", "coredns", "--type", "json", "-p", "[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.cloudprovider.kubernetes.io/uninitialized\",\"value\":\"true\",\"effect\":\"NoSchedule\"}},{\"op\":\"add\",\"path\":\"/spec/template/spec/nodeSelector\",\"value\":{\"node-role.kubernetes.io/control-plane\":\"\"}}]").Run() - if err != nil { - return err - } tolerations := []corev1.Toleration{ { Key: "node.cloudprovider.kubernetes.io/uninitialized", @@ -241,6 +238,12 @@ func (k *KubernetesUtil) setupGCPPodNetwork(ctx context.Context, nodeName, nodeP if err = kubectl.AddTolerationsToDeployment(ctx, tolerations, "coredns"); err != nil { return err } + selectors := map[string]string{ + "node-role.kubernetes.io/control-plane": "", + } + if err = kubectl.AddNodeSelectorsToDeployment(ctx, selectors, "coredns"); err != nil { + return err + } ciliumInstall := exec.CommandContext(ctx, "cilium", "install", "--ipam", "kubernetes", "--ipv4-native-routing-cidr", subnetworkPodCIDR, "--helm-set", "endpointRoutes.enabled=true,tunnel=disabled,encryption.enabled=true,encryption.type=wireguard,l7Proxy=false") diff --git a/bootstrapper/internal/kubernetes/kubernetes_test.go b/bootstrapper/internal/kubernetes/kubernetes_test.go index 564622bdc..fdb01def0 100644 --- a/bootstrapper/internal/kubernetes/kubernetes_test.go +++ b/bootstrapper/internal/kubernetes/kubernetes_test.go @@ -606,6 +606,7 @@ type stubKubectl struct { ApplyErr error createConfigMapErr error AddTolerationsToDeploymentErr error + AddTNodeSelectorsToDeploymentErr error resources []resources.Marshaler kubeconfigs [][]byte @@ -628,6 +629,10 @@ func (s *stubKubectl) AddTolerationsToDeployment(ctx context.Context, toleration return s.AddTolerationsToDeploymentErr } +func (s *stubKubectl) AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string) error { + return s.AddTNodeSelectorsToDeploymentErr +} + type stubKubeconfigReader struct { Kubeconfig []byte ReadErr error