Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-10-01 01:36:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

AB#2234: Introduce AddTolerationsToDeployment

Browse Source 
Add the above function to the different client interfaces.
Update adjacent functions to provided needed ctx and client
objects.
This commit is contained in:
Otto Bittner 2022-07-14 21:15:31 +02:00
parent ad02249b9a
commit 6b6a3ee976
8 changed files with 133 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client.go
View File

@ -14,6 +14,7 @@ import (
"k8s.io/cli-runtime/pkg/resource"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/retry"
)
const fieldManager = "constellation-bootstrapper"
@ -97,6 +98,28 @@ func (c *Client) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap
if err != nil {
return err
}
return nil
}
func (c *Client) AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error {
deployments := c.clientset.AppsV1().Deployments(corev1.NamespaceAll)
// retry resource update if an error occurs
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
result, err := deployments.Get(ctx, name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("Failed to get latest version of Deployment: %v", err)
}
result.Spec.Template.Spec.Tolerations = append(result.Spec.Template.Spec.Tolerations, tolerations...)
if _, err = deployments.Update(ctx, result, metav1.UpdateOptions{}); err != nil {
return err
}
return nil
})
if err != nil {
return err
}
return nil
}

62
bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go
View File

@ -2,6 +2,7 @@ package client
import (
"bytes"
"context"
"errors"
"io"
"net/http"
@ -12,11 +13,12 @@ import (
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
"google.golang.org/protobuf/proto"
apps "k8s.io/api/apps/v1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
k8s "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/api/meta/testrestmapper"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
@ -33,26 +35,26 @@ func TestMain(m *testing.M) {
var (
corev1GV = schema.GroupVersion{Version: "v1"}
nginxDeployment = &apps.Deployment{
TypeMeta: v1.TypeMeta{
nginxDeployment = &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
APIVersion: "apps/v1",
Kind: "Deployment",
},
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "nginx",
},
Name: "my-nginx",
},
Spec: apps.DeploymentSpec{
Spec: appsv1.DeploymentSpec{
Replicas: proto.Int32(3),
Selector: &v1.LabelSelector{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "nginx",
},
},
Template: k8s.PodTemplateSpec{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "nginx",
},
@ -278,3 +280,47 @@ func TestGetObjects(t *testing.T) {
})
}
}
func TestAddTolerationsToDeployment(t *testing.T) {
testCases := map[string]struct {
name string
deployment appsv1.Deployment
tolerations []corev1.Toleration
wantErr bool
}{
"Success": {
name: "test-deployment",
deployment: appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "test-deployment",
},
},
tolerations: []corev1.Toleration{},
},
"Specifying non-existent deployment fails": {
name: "wrong-name",
deployment: appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "test-deployment",
},
},
tolerations: []corev1.Toleration{},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
client := newClientWithFakes(t, map[string]string{}, &tc.deployment)
err := client.AddTolerationsToDeployment(context.Background(), tc.tolerations, tc.name)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
})
}
}

15
bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go
View File

@ -20,6 +20,7 @@ type Client interface {
// GetObjects converts resources into prepared info fields for use in ApplyOneObject.
GetObjects(resources resources.Marshaler) ([]*resource.Info, error)
CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error
}
// clientGenerator can generate new clients from a kubeconfig.
@ -83,3 +84,17 @@ func (k *Kubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMa
return nil
}
func (k *Kubectl) AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error {
client, err := k.clientGenerator.NewClient(k.kubeconfig)
if err != nil {
return err
}
if err = client.AddTolerationsToDeployment(ctx, tolerations, name); err != nil {
return err
}
return nil
}

25
bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl_test.go
View File

@ -17,10 +17,11 @@ func TestMain(m *testing.M) {
}
type stubClient struct {
applyOneObjectErr error
getObjectsInfos []*resource.Info
getObjectsErr error
createConfigMapErr error
applyOneObjectErr error
getObjectsInfos []*resource.Info
getObjectsErr error
createConfigMapErr error
addTolerationsToDeploymentErr error
}
func (s *stubClient) ApplyOneObject(info *resource.Info, forceConflicts bool) error {
@ -35,12 +36,17 @@ func (s *stubClient) CreateConfigMap(ctx context.Context, configMap corev1.Confi
return s.createConfigMapErr
}
func (s *stubClient) AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error {
return s.addTolerationsToDeploymentErr
}
type stubClientGenerator struct {
applyOneObjectErr error
getObjectsInfos []*resource.Info
getObjectsErr error
newClientErr error
createConfigMapErr error
applyOneObjectErr error
getObjectsInfos []*resource.Info
getObjectsErr error
newClientErr error
createConfigMapErr error
addTolerationsToDeploymentErr error
}
func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) {
@ -49,6 +55,7 @@ func (s *stubClientGenerator) NewClient(kubeconfig []byte) (Client, error) {
s.getObjectsInfos,
s.getObjectsErr,
s.createConfigMapErr,
s.addTolerationsToDeploymentErr,
}, s.newClientErr
}

20
bootstrapper/internal/kubernetes/k8sapi/util.go
View File

@ -40,12 +40,12 @@ const (
var providerIDRegex = regexp.MustCompile(`^azure:///subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft.Compute/virtualMachineScaleSets/([^/]+)/virtualMachines/([^/]+)$`)
// Client provides the functionality of `kubectl apply`.
// Client provides the functions to talk to the k8s API.
type Client interface {
Apply(resources resources.Marshaler, forceConflicts bool) error
SetKubeconfig(kubeconfig []byte)
CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error
// TODO: add tolerations
AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error
}
type installer interface {
@ -187,10 +187,10 @@ type SetupPodNetworkInput struct {
}
// SetupPodNetwork sets up the cilium pod network.
func (k *KubernetesUtil) SetupPodNetwork(ctx context.Context, in SetupPodNetworkInput) error {
func (k *KubernetesUtil) SetupPodNetwork(ctx context.Context, in SetupPodNetworkInput, client Client) error {
switch in.CloudProvider {
case "gcp":
return k.setupGCPPodNetwork(ctx, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR)
return k.setupGCPPodNetwork(ctx, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR, client)
case "azure":
return k.setupAzurePodNetwork(ctx, in.ProviderID, in.SubnetworkPodCIDR)
case "qemu":
@ -219,7 +219,7 @@ func (k *KubernetesUtil) setupAzurePodNetwork(ctx context.Context, providerID, s
return nil
}
func (k *KubernetesUtil) setupGCPPodNetwork(ctx context.Context, nodeName, nodePodCIDR, subnetworkPodCIDR string) error {
func (k *KubernetesUtil) setupGCPPodNetwork(ctx context.Context, nodeName, nodePodCIDR, subnetworkPodCIDR string, kubectl Client) error {
out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", kubeConfig, "patch", "node", nodeName, "-p", "{\"spec\":{\"podCIDR\": \""+nodePodCIDR+"\"}}").CombinedOutput()
if err != nil {
err = errors.New(string(out))
@ -231,6 +231,16 @@ func (k *KubernetesUtil) setupGCPPodNetwork(ctx context.Context, nodeName, nodeP
if err != nil {
return err
}
tolerations := []corev1.Toleration{
{
Key: "node.cloudprovider.kubernetes.io/uninitialized",
Value: "true",
Effect: "NoSchedule",
},
}
if err = kubectl.AddTolerationsToDeployment(ctx, tolerations, "coredns"); err != nil {
return err
}
ciliumInstall := exec.CommandContext(ctx, "cilium", "install", "--ipam", "kubernetes", "--ipv4-native-routing-cidr", subnetworkPodCIDR,
"--helm-set", "endpointRoutes.enabled=true,tunnel=disabled,encryption.enabled=true,encryption.type=wireguard,l7Proxy=false")

2
bootstrapper/internal/kubernetes/k8sutil.go
View File

@ -14,7 +14,7 @@ type clusterUtil interface {
InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error
InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, log *logger.Logger) error
JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error
SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput) error
SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput, k8sapi.Client) error
SetupAccessManager(kubectl k8sapi.Client, sshUsers resources.Marshaler) error
SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration resources.Marshaler, secrets resources.Marshaler) error
SetupJoinService(kubectl k8sapi.Client, joinServiceConfiguration resources.Marshaler) error

2
bootstrapper/internal/kubernetes/kubernetes.go
View File

@ -183,7 +183,7 @@ func (k *KubeWrapper) InitCluster(
SubnetworkPodCIDR: subnetworkPodCIDR,
ProviderID: providerID,
}
if err = k.clusterUtil.SetupPodNetwork(ctx, setupPodNetworkInput); err != nil {
if err = k.clusterUtil.SetupPodNetwork(ctx, setupPodNetworkInput, k.client); err != nil {
return nil, fmt.Errorf("setting up pod network: %w", err)
}

11
bootstrapper/internal/kubernetes/kubernetes_test.go
View File

@ -530,7 +530,7 @@ func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, no
return s.initClusterErr
}
func (s *stubClusterUtil) SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput) error {
func (s *stubClusterUtil) SetupPodNetwork(context.Context, k8sapi.SetupPodNetworkInput, k8sapi.Client) error {
return s.setupPodNetworkErr
}
@ -603,8 +603,9 @@ func (s *stubConfigProvider) JoinConfiguration(_ bool) k8sapi.KubeadmJoinYAML {
}
type stubKubectl struct {
ApplyErr error
createConfigMapErr error
ApplyErr error
createConfigMapErr error
AddTolerationsToDeploymentErr error
resources []resources.Marshaler
kubeconfigs [][]byte
@ -623,6 +624,10 @@ func (s *stubKubectl) CreateConfigMap(ctx context.Context, configMap corev1.Conf
return s.createConfigMapErr
}
func (s *stubKubectl) AddTolerationsToDeployment(ctx context.Context, tolerations []corev1.Toleration, name string) error {
return s.AddTolerationsToDeploymentErr
}
type stubKubeconfigReader struct {
Kubeconfig []byte
ReadErr error