AB#2234: Introduce AddTolerationsToDeployment

Add the above function to the different client interfaces.
Update adjacent functions to provided needed ctx and client
objects.

bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go

@@ -14,6 +14,7 @@ import (
 	"k8s.io/cli-runtime/pkg/resource"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
 )
 
 const fieldManager = "constellation-bootstrapper"
@@ -97,6 +98,28 @@ func (c *Client) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap
 	if err != nil {
 		return err
 	}
+	return nil
+}
+
+func (c *Client) AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error {
+	deployments := c.clientset.AppsV1().Deployments(corev1.NamespaceAll)
+
+	// retry resource update if an error occurs
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		result, err := deployments.Get(ctx, name, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("Failed to get latest version of Deployment: %v", err)
+		}
+
+		result.Spec.Template.Spec.Tolerations = append(result.Spec.Template.Spec.Tolerations, tolerations...)
+		if _, err = deployments.Update(ctx, result, metav1.UpdateOptions{}); err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
 
 	return nil
 }

bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go

@@ -2,6 +2,7 @@ package client
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"io"
 	"net/http"
@@ -12,11 +13,12 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 	"google.golang.org/protobuf/proto"
-	apps "k8s.io/api/apps/v1"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	k8s "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/api/meta/testrestmapper"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer/json"
@@ -33,26 +35,26 @@ func TestMain(m *testing.M) {
 
 var (
 	corev1GV        = schema.GroupVersion{Version: "v1"}
-	nginxDeployment = &apps.Deployment{
+	nginxDeployment = &appsv1.Deployment{
-		TypeMeta: v1.TypeMeta{
+		TypeMeta: metav1.TypeMeta{
 			APIVersion: "apps/v1",
 			Kind:       "Deployment",
 		},
-		ObjectMeta: v1.ObjectMeta{
+		ObjectMeta: metav1.ObjectMeta{
 			Labels: map[string]string{
 				"app": "nginx",
 			},
 			Name: "my-nginx",
 		},
-		Spec: apps.DeploymentSpec{
+		Spec: appsv1.DeploymentSpec{
 			Replicas: proto.Int32(3),
-			Selector: &v1.LabelSelector{
+			Selector: &metav1.LabelSelector{
 				MatchLabels: map[string]string{
 					"app": "nginx",
 				},
 			},
 			Template: k8s.PodTemplateSpec{
-				ObjectMeta: v1.ObjectMeta{
+				ObjectMeta: metav1.ObjectMeta{
 					Labels: map[string]string{
 						"app": "nginx",
 					},
@@ -278,3 +280,47 @@ func TestGetObjects(t *testing.T) {
 		})
 	}
 }
+
+func TestAddTolerationsToDeployment(t *testing.T) {
+	testCases := map[string]struct {
+		name        string
+		deployment  appsv1.Deployment
+		tolerations []corev1.Toleration
+		wantErr     bool
+	}{
+		"Success": {
+			name: "test-deployment",
+			deployment: appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-deployment",
+				},
+			},
+			tolerations: []corev1.Toleration{},
+		},
+		"Specifying non-existent deployment fails": {
+			name: "wrong-name",
+			deployment: appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-deployment",
+				},
+			},
+			tolerations: []corev1.Toleration{},
+			wantErr:     true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			client := newClientWithFakes(t, map[string]string{}, &tc.deployment)
+			err := client.AddTolerationsToDeployment(context.Background(), tc.tolerations, tc.name)
+			if tc.wantErr {
+				assert.Error(err)
+				return
+			}
+			require.NoError(err)
+		})
+	}
+}

bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go

@@ -20,6 +20,7 @@ type Client interface {
 	// GetObjects converts resources into prepared info fields for use in ApplyOneObject.
 	GetObjects(resources resources.Marshaler) ([]*resource.Info, error)
 	CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
+	AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error
 }
 
 // clientGenerator can generate new clients from a kubeconfig.
@@ -83,3 +84,17 @@ func (k *Kubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMa
 
 	return nil
 }
+
+func (k *Kubectl) AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error {
+	client, err := k.clientGenerator.NewClient(k.kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	if err = client.AddTolerationsToDeployment(ctx, tolerations, name); err != nil {
+		return err
+	}
+
+	return nil
+}
+

bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go

@@ -21,6 +21,7 @@ type stubClient struct {
 	getObjectsInfos                []*resource.Info
 	getObjectsErr                  error
 	createConfigMapErr             error
+	addTolerationsToDeploymentErr  error
 }
 
 func (s *stubClient) ApplyOneObject(info *resource.Info, forceConflicts bool) error {
@@ -35,12 +36,17 @@ func (s *stubClient) CreateConfigMap(ctx context.Context, configMap corev1.Confi
 	return s.createConfigMapErr
 }
 
+func (s *stubClient) AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error {
+	return s.addTolerationsToDeploymentErr
+}
+
 type stubClientGenerator struct {
 	applyOneObjectErr              error
 	getObjectsInfos                []*resource.Info
 	getObjectsErr                  error
 	newClientErr                   error
 	createConfigMapErr             error
+	addTolerationsToDeploymentErr  error
 }
 
 func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) {
@@ -49,6 +55,7 @@ func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) {
 		s.getObjectsInfos,
 		s.getObjectsErr,
 		s.createConfigMapErr,
+		s.addTolerationsToDeploymentErr,
 	}, s.newClientErr
 }
 

bootstrapper/internal/kubernetes/k8sapi/util.go

@@ -40,12 +40,12 @@ const (
 
 var providerIDRegex = regexp.MustCompile(`^azure:///subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft.Compute/virtualMachineScaleSets/([^/]+)/virtualMachines/([^/]+)$`)
 
-// Client provides the functionality of `kubectl apply`.
+// Client provides the functions to talk to the k8s API.
 type Client interface {
 	Apply(resources resources.Marshaler, forceConflicts bool) error
 	SetKubeconfig(kubeconfig []byte)
 	CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
-	// TODO: add tolerations
+	AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error
 }
 
 type installer interface {
@@ -187,10 +187,10 @@ type SetupPodNetworkInput struct {
 }
 
 // SetupPodNetwork sets up the cilium pod network.
-func (k *KubernetesUtil) SetupPodNetwork(ctx context.Context, in SetupPodNetworkInput) error {
+func (k *KubernetesUtil) SetupPodNetwork(ctx context.Context, in SetupPodNetworkInput, client Client) error {
 	switch in.CloudProvider {
 	case "gcp":
-		return k.setupGCPPodNetwork(ctx, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR)
+		return k.setupGCPPodNetwork(ctx, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR, client)
 	case "azure":
 		return k.setupAzurePodNetwork(ctx, in.ProviderID, in.SubnetworkPodCIDR)
 	case "qemu":
@@ -219,7 +219,7 @@ func (k *KubernetesUtil) setupAzurePodNetwork(ctx context.Context, providerID, s
 	return nil
 }
 
-func (k *KubernetesUtil) setupGCPPodNetwork(ctx context.Context, nodeName, nodePodCIDR, subnetworkPodCIDR string) error {
+func (k *KubernetesUtil) setupGCPPodNetwork(ctx context.Context, nodeName, nodePodCIDR, subnetworkPodCIDR string, kubectl Client) error {
 	out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", kubeConfig, "patch", "node", nodeName, "-p", "{\"spec\":{\"podCIDR\": \""+nodePodCIDR+"\"}}").CombinedOutput()
 	if err != nil {
 		err = errors.New(string(out))
@@ -231,6 +231,16 @@ func (k *KubernetesUtil) setupGCPPodNetwork(ctx context.Context, nodeName, nodeP
 	if err != nil {
 		return err
 	}
+	tolerations := []corev1.Toleration{
+		{
+			Key:    "node.cloudprovider.kubernetes.io/uninitialized",
+			Value:  "true",
+			Effect: "NoSchedule",
+		},
+	}
+	if err = kubectl.AddTolerationsToDeployment(ctx, tolerations, "coredns"); err != nil {
+		return err
+	}
 
 	ciliumInstall := exec.CommandContext(ctx, "cilium", "install", "--ipam", "kubernetes", "--ipv4-native-routing-cidr", subnetworkPodCIDR,
 		"--helm-set", "endpointRoutes.enabled=true,tunnel=disabled,encryption.enabled=true,encryption.type=wireguard,l7Proxy=false")

bootstrapper/internal/kubernetes/k8sutil.go

@@ -14,7 +14,7 @@ type clusterUtil interface {
 	InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error
 	InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, log *logger.Logger) error
 	JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error
-	SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput) error
+	SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput, k8sapi.Client) error
 	SetupAccessManager(kubectl k8sapi.Client, sshUsers resources.Marshaler) error
 	SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration resources.Marshaler, secrets resources.Marshaler) error
 	SetupJoinService(kubectl k8sapi.Client, joinServiceConfiguration resources.Marshaler) error

bootstrapper/internal/kubernetes/kubernetes.go

@@ -183,7 +183,7 @@ func (k *KubeWrapper) InitCluster(
 		SubnetworkPodCIDR: subnetworkPodCIDR,
 		ProviderID:        providerID,
 	}
-	if err = k.clusterUtil.SetupPodNetwork(ctx, setupPodNetworkInput); err != nil {
+	if err = k.clusterUtil.SetupPodNetwork(ctx, setupPodNetworkInput, k.client); err != nil {
 		return nil, fmt.Errorf("setting up pod network: %w", err)
 	}
 

bootstrapper/internal/kubernetes/kubernetes_test.go

@@ -530,7 +530,7 @@ func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, no
 	return s.initClusterErr
 }
 
-func (s *stubClusterUtil) SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput) error {
+func (s *stubClusterUtil) SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput, k8sapi.Client) error {
 	return s.setupPodNetworkErr
 }
 
@@ -605,6 +605,7 @@ func (s *stubConfigProvider) JoinConfiguration(_ bool) k8sapi.KubeadmJoinYAML {
 type stubKubectl struct {
 	ApplyErr                         error
 	createConfigMapErr               error
+	AddTolerationsToDeploymentErr    error
 
 	resources   []resources.Marshaler
 	kubeconfigs [][]byte
@@ -623,6 +624,10 @@ func (s *stubKubectl) CreateConfigMap(ctx context.Context, configMap corev1.Conf
 	return s.createConfigMapErr
 }
 
+func (s *stubKubectl) AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error {
+	return s.AddTolerationsToDeploymentErr
+}
+
 type stubKubeconfigReader struct {
 	Kubeconfig []byte
 	ReadErr    error