Add Secrets / Volumes / VolumeMounts / Env to cluster-autoscaler deployment

coordinator/core/cloud_provider.go

@@ -92,6 +92,14 @@ type CloudNodeManager interface {
 type ClusterAutoscaler interface {
 	// Name returns the cloud-provider name as used by k8s cluster-autoscaler.
 	Name() string
+	// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler.
+	Secrets(instance Instance, cloudServiceAccountURI string) (resources.Secrets, error)
+	// Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler.
+	Volumes() []k8s.Volume
+	// VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler.
+	VolumeMounts() []k8s.VolumeMount
+	// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler.
+	Env() []k8s.EnvVar
 	// Supported is used to determine if cluster autoscaler is implemented for this cloud provider.
 	Supported() bool
 }
@@ -238,6 +246,26 @@ func (f *ClusterAutoscalerFake) Name() string {
 	return "fake"
 }
 
+// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler.
+func (f *ClusterAutoscalerFake) Secrets(instance Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
+	return resources.Secrets{}, nil
+}
+
+// Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler.
+func (f *ClusterAutoscalerFake) Volumes() []k8s.Volume {
+	return []k8s.Volume{}
+}
+
+// VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler.
+func (f *ClusterAutoscalerFake) VolumeMounts() []k8s.VolumeMount {
+	return []k8s.VolumeMount{}
+}
+
+// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler.
+func (f *ClusterAutoscalerFake) Env() []k8s.EnvVar {
+	return []k8s.EnvVar{}
+}
+
 func (f *ClusterAutoscalerFake) Supported() bool {
 	return false
 }

coordinator/core/cluster.go

@@ -23,6 +23,7 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountUR
 	var instance Instance
 	var ccmConfigMaps resources.ConfigMaps
 	var ccmSecrets resources.Secrets
+	var caSecrets resources.Secrets
 	var err error
 	nodeIP := coordinatorVPNIP.String()
 	if c.metadata.Supported() {
@@ -56,6 +57,13 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountUR
 			return nil, err
 		}
 	}
+	if c.clusterAutoscaler.Supported() {
+		caSecrets, err = c.clusterAutoscaler.Secrets(instance, cloudServiceAccountURI)
+		if err != nil {
+			c.zaplogger.Error("Defining Secrets for cluster-autoscaler failed", zap.Error(err))
+			return nil, err
+		}
+	}
 
 	c.zaplogger.Info("Initializing cluster")
 	joinCommand, err := c.kube.InitCluster(kubernetes.InitClusterInput{
@@ -65,6 +73,10 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountUR
 		ProviderID:                         providerID,
 		SupportClusterAutoscaler:           c.clusterAutoscaler.Supported(),
 		AutoscalingCloudprovider:           c.clusterAutoscaler.Name(),
+		AutoscalingSecrets:                 caSecrets,
+		AutoscalingVolumes:                 c.clusterAutoscaler.Volumes(),
+		AutoscalingVolumeMounts:            c.clusterAutoscaler.VolumeMounts(),
+		AutoscalingEnv:                     c.clusterAutoscaler.Env(),
 		AutoscalingNodeGroups:              autoscalingNodeGroups,
 		SupportsCloudControllerManager:     c.cloudControllerManager.Supported(),
 		CloudControllerManagerName:         c.cloudControllerManager.Name(),

coordinator/core/cluster_test.go

@@ -462,14 +462,35 @@ func (s *stubCloudNodeManager) Supported() bool {
 }
 
 type stubClusterAutoscaler struct {
-	nameRes      string
+	nameRes        string
-	supportedRes bool
+	supportedRes   bool
+	secretsRes     resources.Secrets
+	secretsErr     error
+	volumesRes     []k8s.Volume
+	volumeMountRes []k8s.VolumeMount
+	envRes         []k8s.EnvVar
 }
 
 func (s *stubClusterAutoscaler) Name() string {
 	return s.nameRes
 }
 
+func (s *stubClusterAutoscaler) Secrets(instance Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
+	return s.secretsRes, s.secretsErr
+}
+
+func (s *stubClusterAutoscaler) Volumes() []k8s.Volume {
+	return s.volumesRes
+}
+
+func (s *stubClusterAutoscaler) VolumeMounts() []k8s.VolumeMount {
+	return s.volumeMountRes
+}
+
+func (s *stubClusterAutoscaler) Env() []k8s.EnvVar {
+	return s.envRes
+}
+
 func (s *stubClusterAutoscaler) Supported() bool {
 	return s.supportedRes
 }

coordinator/kubernetes/cluster_input.go

@@ -14,6 +14,10 @@ type InitClusterInput struct {
 	SupportClusterAutoscaler           bool
 	AutoscalingCloudprovider           string
 	AutoscalingNodeGroups              []string
+	AutoscalingSecrets                 resources.Secrets
+	AutoscalingVolumes                 []k8s.Volume
+	AutoscalingVolumeMounts            []k8s.VolumeMount
+	AutoscalingEnv                     []k8s.EnvVar
 	SupportsCloudControllerManager     bool
 	CloudControllerManagerName         string
 	CloudControllerManagerImage        string

coordinator/kubernetes/k8sapi/resources/cluster_autoscaler.go

@@ -21,7 +21,8 @@ type autoscalerDeployment struct {
 	Deployment          apps.Deployment
 }
 
-func NewDefaultAutoscalerDeployment() *autoscalerDeployment {
+// NewDefaultAutoscalerDeployment creates a new *autoscalerDeployment, customized for the CSP.
+func NewDefaultAutoscalerDeployment(extraVolumes []k8s.Volume, extraVolumeMounts []k8s.VolumeMount, env []k8s.EnvVar) *autoscalerDeployment {
 	return &autoscalerDeployment{
 		PodDisruptionBudget: policy.PodDisruptionBudget{
 			TypeMeta: v1.TypeMeta{
@@ -433,7 +434,7 @@ func NewDefaultAutoscalerDeployment() *autoscalerDeployment {
 						Containers: []k8s.Container{
 							{
 								Name:            "cluster-autoscaler",
-								Image:           "k8s.gcr.io/autoscaling/cluster-autoscaler:v1.21.1",
+								Image:           "k8s.gcr.io/autoscaling/cluster-autoscaler:v1.23.0",
 								ImagePullPolicy: k8s.PullIfNotPresent,
 								LivenessProbe: &k8s.Probe{
 									ProbeHandler: k8s.ProbeHandler{
@@ -448,8 +449,11 @@ func NewDefaultAutoscalerDeployment() *autoscalerDeployment {
 										ContainerPort: 8085,
 									},
 								},
+								VolumeMounts: extraVolumeMounts,
+								Env:          env,
 							},
 						},
+						Volumes:            extraVolumes,
 						ServiceAccountName: "constellation-cluster-autoscaler",
 						Tolerations: []k8s.Toleration{
 							{

coordinator/kubernetes/k8sapi/resources/cluster_autoscaler_test.go

@@ -11,7 +11,7 @@ func TestAutoscalerDeploymentMarshalUnmarshal(t *testing.T) {
 	require := require.New(t)
 	assert := assert.New(t)
 
-	autoscalerDepl := NewDefaultAutoscalerDeployment()
+	autoscalerDepl := NewDefaultAutoscalerDeployment(nil, nil, nil)
 
 	data, err := autoscalerDepl.Marshal()
 	require.NoError(err)
@@ -27,7 +27,7 @@ func TestAutoscalerDeploymentWithCommandMarshalUnmarshal(t *testing.T) {
 	require := require.New(t)
 	assert := assert.New(t)
 
-	autoscalerDepl := NewDefaultAutoscalerDeployment()
+	autoscalerDepl := NewDefaultAutoscalerDeployment(nil, nil, nil)
 	autoscalerDepl.SetAutoscalerCommand("someProvider", []string{"group1", "group2"})
 
 	data, err := autoscalerDepl.Marshal()

coordinator/kubernetes/k8sapi/util.go

@@ -25,7 +25,7 @@ type ClusterUtil interface {
 	InitCluster(initConfig []byte) (*kubeadm.BootstrapTokenDiscovery, error)
 	JoinCluster(joinConfig []byte) error
 	SetupPodNetwork(kubectl Client, podNetworkConfiguration resources.Marshaler) error
-	SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler) error
+	SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler, secrets resources.Marshaler) error
 	SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error
 	SetupCloudNodeManager(kubectl Client, cloudNodeManagerConfiguration resources.Marshaler) error
 	RestartKubelet() error
@@ -113,7 +113,10 @@ func (k *KubernetesUtil) SetupPodNetwork(kubectl Client, podNetworkConfiguration
 }
 
 // SetupAutoscaling deploys the k8s cluster autoscaler.
-func (k *KubernetesUtil) SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler) error {
+func (k *KubernetesUtil) SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler, secrets resources.Marshaler) error {
+	if err := kubectl.Apply(secrets, true); err != nil {
+		return fmt.Errorf("applying cluster-autoscaler Secrets failed: %w", err)
+	}
 	return kubectl.Apply(clusterAutoscalerConfiguration, true)
 }
 

coordinator/kubernetes/kubernetes.go

@@ -92,9 +92,9 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
 	}
 
 	if in.SupportClusterAutoscaler {
-		clusterAutoscalerConfiguration := resources.NewDefaultAutoscalerDeployment()
+		clusterAutoscalerConfiguration := resources.NewDefaultAutoscalerDeployment(in.AutoscalingVolumes, in.AutoscalingVolumeMounts, in.AutoscalingEnv)
 		clusterAutoscalerConfiguration.SetAutoscalerCommand(in.AutoscalingCloudprovider, in.AutoscalingNodeGroups)
-		if err := k.clusterUtil.SetupAutoscaling(k.client, clusterAutoscalerConfiguration); err != nil {
+		if err := k.clusterUtil.SetupAutoscaling(k.client, clusterAutoscalerConfiguration, in.AutoscalingSecrets); err != nil {
 			return nil, fmt.Errorf("failed to setup cluster-autoscaler: %w", err)
 		}
 	}

coordinator/kubernetes/kubernetes_test.go

@@ -43,7 +43,7 @@ func (s *stubClusterUtil) SetupPodNetwork(kubectl k8sapi.Client, podNetworkConfi
 	return s.setupPodNetworkErr
 }
 
-func (s *stubClusterUtil) SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration resources.Marshaler) error {
+func (s *stubClusterUtil) SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration resources.Marshaler, secrets resources.Marshaler) error {
 	return s.setupAutoscalingError
 }