Cloud providers: Extend CCM with ExtraArgs / ConfigMaps / Secrets / Volumes / VolumeMounts and provide CloudServiceAccountURI

coordinator/core/cloud_provider.go

@@ -6,7 +6,9 @@ import (
 	"fmt"
 	"net"
 
+	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
 	"github.com/edgelesssys/constellation/coordinator/role"
+	k8s "k8s.io/api/core/v1"
 )
 
 var ErrUnimplemented = errors.New("unimplemented")
@@ -52,6 +54,21 @@ type CloudControllerManager interface {
 	Path() string
 	// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager).
 	Name() string
+	// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command.
+	ExtraArgs() []string
+	// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
+	// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
+	ConfigMaps(instance Instance) (resources.ConfigMaps, error)
+	// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
+	// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
+	Secrets(instance Instance, cloudServiceAccountURI string) (resources.Secrets, error)
+	// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager.
+	// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ .
+	Volumes() []k8s.Volume
+	// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager.
+	VolumeMounts() []k8s.VolumeMount
+	// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager.
+	Env() []k8s.EnvVar
 	// PrepareInstance is called on every instance before deploying the cloud-controller-manager.
 	// Allows for cloud-provider specific hooks.
 	PrepareInstance(instance Instance, vpnIP string) error
@@ -153,6 +170,30 @@ func (f *CloudControllerManagerFake) Name() string {
 	return "fake"
 }
 
+func (f *CloudControllerManagerFake) ExtraArgs() []string {
+	return []string{}
+}
+
+func (f *CloudControllerManagerFake) ConfigMaps(instance Instance) (resources.ConfigMaps, error) {
+	return []*k8s.ConfigMap{}, nil
+}
+
+func (f *CloudControllerManagerFake) Secrets(instance Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
+	return []*k8s.Secret{}, nil
+}
+
+func (f *CloudControllerManagerFake) Volumes() []k8s.Volume {
+	return []k8s.Volume{}
+}
+
+func (f *CloudControllerManagerFake) VolumeMounts() []k8s.VolumeMount {
+	return []k8s.VolumeMount{}
+}
+
+func (f *CloudControllerManagerFake) Env() []k8s.EnvVar {
+	return []k8s.EnvVar{}
+}
+
 func (f *CloudControllerManagerFake) PrepareInstance(instance Instance, vpnIP string) error {
 	return nil
 }

coordinator/core/cluster.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/edgelesssys/constellation/coordinator/kubernetes"
+	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
 	"github.com/edgelesssys/constellation/coordinator/role"
 	"go.uber.org/zap"
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
@@ -16,12 +17,17 @@ func (c *Core) GetK8sJoinArgs() (*kubeadm.BootstrapTokenDiscovery, error) {
 }
 
 // InitCluster initializes the cluster, stores the join args, and returns the kubeconfig.
-func (c *Core) InitCluster(autoscalingNodeGroups []string) ([]byte, error) {
+func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountURI string) ([]byte, error) {
 	var nodeName string
 	var providerID string
+	var instance Instance
+	var ccmConfigMaps resources.ConfigMaps
+	var ccmSecrets resources.Secrets
+	var err error
+	nodeIP := coordinatorVPNIP.String()
+	var err error
 	if c.metadata.Supported() {
-		var err error
+		instance, err = c.metadata.Self(context.TODO())
-		instance, err := c.metadata.Self(context.TODO())
 		if err != nil {
 			c.zaplogger.Error("Retrieving own instance metadata failed", zap.Error(err))
 			return nil, err
@@ -33,25 +39,41 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string) ([]byte, error) {
 	}
 	if c.cloudControllerManager.Supported() && c.metadata.Supported() {
 		c.zaplogger.Info("Preparing node for cloud-controller-manager")
-
 		if err := PrepareInstanceForCCM(context.TODO(), c.metadata, c.cloudControllerManager, coordinatorVPNIP.String()); err != nil {
 			c.zaplogger.Error("Preparing node for CCM failed", zap.Error(err))
 			return nil, err
 		}
+		ccmConfigMaps, err = c.cloudControllerManager.ConfigMaps(instance)
+		if err != nil {
+			c.zaplogger.Error("Defining ConfigMaps for CCM failed", zap.Error(err))
+			return nil, err
+		}
+		ccmSecrets, err = c.cloudControllerManager.Secrets(instance, cloudServiceAccountURI)
+		if err != nil {
+			c.zaplogger.Error("Defining Secrets for CCM failed", zap.Error(err))
+			return nil, err
+		}
 	}
 
 	c.zaplogger.Info("Initializing cluster")
 	joinCommand, err := c.kube.InitCluster(kubernetes.InitClusterInput{
-		APIServerAdvertiseIP:           coordinatorVPNIP.String(),
+		APIServerAdvertiseIP:               coordinatorVPNIP.String(),
-		NodeName:                       k8sCompliantHostname(nodeName),
+		NodeIP:                             nodeIP,
-		ProviderID:                     providerID,
+		NodeName:                           k8sCompliantHostname(nodeName),
-		SupportClusterAutoscaler:       c.clusterAutoscaler.Supported(),
+		ProviderID:                         providerID,
-		AutoscalingCloudprovider:       c.clusterAutoscaler.Name(),
+		SupportClusterAutoscaler:           c.clusterAutoscaler.Supported(),
-		AutoscalingNodeGroups:          autoscalingNodeGroups,
+		AutoscalingCloudprovider:           c.clusterAutoscaler.Name(),
-		SupportsCloudControllerManager: c.cloudControllerManager.Supported(),
+		AutoscalingNodeGroups:              autoscalingNodeGroups,
-		CloudControllerManagerName:     c.cloudControllerManager.Name(),
+		SupportsCloudControllerManager:     c.cloudControllerManager.Supported(),
-		CloudControllerManagerImage:    c.cloudControllerManager.Image(),
+		CloudControllerManagerName:         c.cloudControllerManager.Name(),
-		CloudControllerManagerPath:     c.cloudControllerManager.Path(),
+		CloudControllerManagerImage:        c.cloudControllerManager.Image(),
+		CloudControllerManagerPath:         c.cloudControllerManager.Path(),
+		CloudControllerManagerExtraArgs:    c.cloudControllerManager.ExtraArgs(),
+		CloudControllerManagerConfigMaps:   ccmConfigMaps,
+		CloudControllerManagerSecrets:      ccmSecrets,
+		CloudControllerManagerVolumes:      c.cloudControllerManager.Volumes(),
+		CloudControllerManagerVolumeMounts: c.cloudControllerManager.VolumeMounts(),
+		CloudControllerManagerEnv:          c.cloudControllerManager.Env(),
 	})
 	if err != nil {
 		c.zaplogger.Error("Initializing cluster failed", zap.Error(err))

coordinator/core/cluster_test.go

@@ -7,9 +7,11 @@ import (
 
 	"github.com/edgelesssys/constellation/coordinator/attestation/vtpm"
 	"github.com/edgelesssys/constellation/coordinator/kubernetes"
+	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
+	k8s "k8s.io/api/core/v1"
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
 )
 
@@ -36,6 +38,7 @@ func TestInitCluster(t *testing.T) {
 				SupportsCloudControllerManager: false,
 				SupportClusterAutoscaler:       false,
 				AutoscalingNodeGroups:          []string{"someNodeGroup"},
+				CloudControllerManagerEnv:      []k8s.EnvVar{},
 			},
 			expectErr: false,
 		},
@@ -56,6 +59,7 @@ func TestInitCluster(t *testing.T) {
 				ProviderID:                     "fake://providerid",
 				SupportsCloudControllerManager: false,
 				SupportClusterAutoscaler:       false,
+				CloudControllerManagerEnv:      []k8s.EnvVar{},
 			},
 			expectErr: false,
 		},
@@ -85,6 +89,7 @@ func TestInitCluster(t *testing.T) {
 				SupportClusterAutoscaler:       true,
 				AutoscalingCloudprovider:       "some-name",
 				AutoscalingNodeGroups:          []string{"someNodeGroup"},
+				CloudControllerManagerEnv:      []k8s.EnvVar{},
 			},
 			expectErr: false,
 		},
@@ -106,6 +111,7 @@ func TestInitCluster(t *testing.T) {
 				CloudControllerManagerName:     "some-name",
 				CloudControllerManagerImage:    "someImage",
 				CloudControllerManagerPath:     "/some/path",
+				CloudControllerManagerEnv:      []k8s.EnvVar{},
 			},
 			expectErr: false,
 		},
@@ -136,6 +142,7 @@ func TestInitCluster(t *testing.T) {
 			expectErr: false,
 			expectedInitClusterInput: kubernetes.InitClusterInput{
 				APIServerAdvertiseIP: "10.118.0.1",
+				CloudControllerManagerEnv: []k8s.EnvVar{},
 			},
 		},
 		"getting kubeconfig fail detected": {
@@ -162,7 +169,7 @@ func TestInitCluster(t *testing.T) {
 			core, err := NewCore(&stubVPN{}, &tc.cluster, &tc.metadata, &tc.cloudControllerManager, &tc.clusterAutoscaler, zapLogger, vtpm.OpenSimulatedTPM, nil)
 			require.NoError(err)
 
-			kubeconfig, err := core.InitCluster(tc.autoscalingNodeGroups)
+			kubeconfig, err := core.InitCluster(tc.autoscalingNodeGroups, "cloud-service-account-uri")
 
 			if tc.expectErr {
 				assert.Error(err)
@@ -368,6 +375,13 @@ type stubCloudControllerManager struct {
 	pathRes            string
 	nameRes            string
 	prepareInstanceRes error
+	extraArgsRes       []string
+	configMapsRes      resources.ConfigMaps
+	configMapsErr      error
+	secretsRes         resources.Secrets
+	secretsErr         error
+	volumesRes         []k8s.Volume
+	volumeMountRes     []k8s.VolumeMount
 	supportedRes       bool
 
 	prepareInstanceRequests []prepareInstanceRequest
@@ -393,6 +407,30 @@ func (s *stubCloudControllerManager) PrepareInstance(instance Instance, vpnIP st
 	return s.prepareInstanceRes
 }
 
+func (s *stubCloudControllerManager) ExtraArgs() []string {
+	return s.extraArgsRes
+}
+
+func (s *stubCloudControllerManager) ConfigMaps(instance Instance) (resources.ConfigMaps, error) {
+	return s.configMapsRes, s.configMapsErr
+}
+
+func (s *stubCloudControllerManager) Secrets(instance Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
+	return s.secretsRes, s.secretsErr
+}
+
+func (s *stubCloudControllerManager) Volumes() []k8s.Volume {
+	return s.volumesRes
+}
+
+func (s *stubCloudControllerManager) VolumeMounts() []k8s.VolumeMount {
+	return s.volumeMountRes
+}
+
+func (s *stubCloudControllerManager) Env() []k8s.EnvVar {
+	return []k8s.EnvVar{}
+}
+
 func (s *stubCloudControllerManager) Supported() bool {
 	return s.supportedRes
 }

coordinator/kubernetes/cluster_input.go

@@ -1,15 +1,27 @@
 package kubernetes
 
+import (
+	"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
+	k8s "k8s.io/api/core/v1"
+)
+
 // InitClusterInput collects the arguments to initialize a new cluster.
 type InitClusterInput struct {
-	APIServerAdvertiseIP           string
+	APIServerAdvertiseIP               string
-	NodeName                       string
+	NodeIP                             string
-	ProviderID                     string
+	NodeName                           string
-	SupportClusterAutoscaler       bool
+	ProviderID                         string
-	AutoscalingCloudprovider       string
+	SupportClusterAutoscaler           bool
-	AutoscalingNodeGroups          []string
+	AutoscalingCloudprovider           string
-	SupportsCloudControllerManager bool
+	AutoscalingNodeGroups              []string
-	CloudControllerManagerName     string
+	SupportsCloudControllerManager     bool
-	CloudControllerManagerImage    string
+	CloudControllerManagerName         string
-	CloudControllerManagerPath     string
+	CloudControllerManagerImage        string
+	CloudControllerManagerPath         string
+	CloudControllerManagerExtraArgs    []string
+	CloudControllerManagerConfigMaps   resources.ConfigMaps
+	CloudControllerManagerSecrets      resources.Secrets
+	CloudControllerManagerVolumes      []k8s.Volume
+	CloudControllerManagerVolumeMounts []k8s.VolumeMount
+	CloudControllerManagerEnv          []k8s.EnvVar
 }

coordinator/kubernetes/k8sapi/kubectl/client/client_test.go

@@ -250,7 +250,7 @@ func TestGetObjects(t *testing.T) {
 			expectErr:         false,
 		},
 		"GetObjects works on cloud-controller-manager deployment": {
-			expectedResources: resources.NewDefaultCloudControllerManagerDeployment("someProvider", "someImage", "somePath"),
+			expectedResources: resources.NewDefaultCloudControllerManagerDeployment("someProvider", "someImage", "somePath", nil, nil, nil, nil),
 			resourcesYAML:     string(nginxDeplYAML),
 			expectErr:         false,
 		},

coordinator/kubernetes/k8sapi/resources/cloud_controller_manager.go

@@ -9,6 +9,8 @@ import (
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const defaultCIDR = "10.244.0.0/16"
+
 type cloudControllerManagerDeployment struct {
 	ServiceAccount     k8s.ServiceAccount
 	ClusterRoleBinding rbac.ClusterRoleBinding
@@ -19,7 +21,58 @@ type cloudControllerManagerDeployment struct {
 // https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/admin/cloud/ccm-example.yaml
 // https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/#cloud-controller-manager
 
-func NewDefaultCloudControllerManagerDeployment(cloudProvider string, image string, path string) *cloudControllerManagerDeployment {
+// NewDefaultCloudControllerManagerDeployment creates a new *cloudControllerManagerDeployment, customized for the CSP.
+func NewDefaultCloudControllerManagerDeployment(cloudProvider, image, path string, extraArgs []string, extraVolumes []k8s.Volume, extraVolumeMounts []k8s.VolumeMount, env []k8s.EnvVar) *cloudControllerManagerDeployment {
+	command := []string{
+		path,
+		fmt.Sprintf("--cloud-provider=%s", cloudProvider),
+		"--leader-elect=true",
+		"--allocate-node-cidrs=false",
+		"--configure-cloud-routes=false",
+		fmt.Sprintf("--cluster-cidr=%s", defaultCIDR),
+		"-v=2",
+	}
+	command = append(command, extraArgs...)
+	volumes := []k8s.Volume{
+		{
+			Name: "etckubernetes",
+			VolumeSource: k8s.VolumeSource{
+				HostPath: &k8s.HostPathVolumeSource{Path: "/etc/kubernetes"},
+			},
+		},
+		{
+			Name: "etcssl",
+			VolumeSource: k8s.VolumeSource{
+				HostPath: &k8s.HostPathVolumeSource{Path: "/etc/ssl"},
+			},
+		},
+		{
+			Name: "etcpki",
+			VolumeSource: k8s.VolumeSource{
+				HostPath: &k8s.HostPathVolumeSource{Path: "/etc/pki"},
+			},
+		},
+	}
+	volumes = append(volumes, extraVolumes...)
+	volumeMounts := []k8s.VolumeMount{
+		{
+			MountPath: "/etc/kubernetes",
+			Name:      "etckubernetes",
+			ReadOnly:  true,
+		},
+		{
+			MountPath: "/etc/ssl",
+			Name:      "etcssl",
+			ReadOnly:  true,
+		},
+		{
+			MountPath: "/etc/pki",
+			Name:      "etcpki",
+			ReadOnly:  true,
+		},
+	}
+	volumeMounts = append(volumeMounts, extraVolumeMounts...)
+
 	return &cloudControllerManagerDeployment{
 		ServiceAccount: k8s.ServiceAccount{
 			TypeMeta: meta.TypeMeta{
@@ -80,69 +133,14 @@ func NewDefaultCloudControllerManagerDeployment(cloudProvider string, image stri
 						ServiceAccountName: "cloud-controller-manager",
 						Containers: []k8s.Container{
 							{
-								Name:  "cloud-controller-manager",
+								Name:         "cloud-controller-manager",
-								Image: image,
+								Image:        image,
-								Command: []string{
+								Command:      command,
-									path,
+								VolumeMounts: volumeMounts,
-									fmt.Sprintf("--cloud-provider=%s", cloudProvider),
+								Env:          env,
-									"--leader-elect=true",
-									"--allocate-node-cidrs=false",
-									"--configure-cloud-routes=false",
-									"--controllers=cloud-node,cloud-node-lifecycle",
-									"--use-service-account-credentials",
-									"--cluster-cidr=10.244.0.0/16",
-									"-v=2",
-								},
-								VolumeMounts: []k8s.VolumeMount{
-									{
-										MountPath: "/etc/kubernetes",
-										Name:      "etckubernetes",
-										ReadOnly:  true,
-									},
-									{
-										MountPath: "/etc/ssl",
-										Name:      "etcssl",
-										ReadOnly:  true,
-									},
-									{
-										MountPath: "/etc/pki",
-										Name:      "etcpki",
-										ReadOnly:  true,
-									},
-									{
-										MountPath: "/etc/gce.conf",
-										Name:      "gceconf",
-										ReadOnly:  true,
-									},
-								},
-							},
-						},
-						Volumes: []k8s.Volume{
-							{
-								Name: "etckubernetes",
-								VolumeSource: k8s.VolumeSource{
-									HostPath: &k8s.HostPathVolumeSource{Path: "/etc/kubernetes"},
-								},
-							},
-							{
-								Name: "etcssl",
-								VolumeSource: k8s.VolumeSource{
-									HostPath: &k8s.HostPathVolumeSource{Path: "/etc/ssl"},
-								},
-							},
-							{
-								Name: "etcpki",
-								VolumeSource: k8s.VolumeSource{
-									HostPath: &k8s.HostPathVolumeSource{Path: "/etc/pki"},
-								},
-							},
-							{
-								Name: "gceconf",
-								VolumeSource: k8s.VolumeSource{
-									HostPath: &k8s.HostPathVolumeSource{Path: "/etc/gce.conf"},
-								},
 							},
 						},
+						Volumes: volumes,
 						Tolerations: []k8s.Toleration{
 							{
 								Key:    "node.cloudprovider.kubernetes.io/uninitialized",

coordinator/kubernetes/k8sapi/resources/cloud_controller_manager_test.go

@@ -5,13 +5,14 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	k8s "k8s.io/api/core/v1"
 )
 
 func TestCloudControllerMarshalUnmarshal(t *testing.T) {
 	require := require.New(t)
 	assert := assert.New(t)
 
-	cloudControllerManagerDepl := NewDefaultCloudControllerManagerDeployment("dummy-cloudprovider", "some-image:latest", "/dummy_path")
+	cloudControllerManagerDepl := NewDefaultCloudControllerManagerDeployment("dummy-cloudprovider", "some-image:latest", "/dummy_path", []string{}, []k8s.Volume{}, []k8s.VolumeMount{}, nil)
 	data, err := cloudControllerManagerDepl.Marshal()
 	require.NoError(err)
 

coordinator/kubernetes/k8sapi/resources/configmaps.go

@@ -0,0 +1,18 @@
+package resources
+
+import (
+	k8s "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// ConfigMaps represent a list of k8s ConfigMap.
+type ConfigMaps []*k8s.ConfigMap
+
+// Marshal marshals config maps into multiple YAML documents.
+func (s ConfigMaps) Marshal() ([]byte, error) {
+	objects := make([]runtime.Object, len(s))
+	for i := range s {
+		objects[i] = s[i]
+	}
+	return MarshalK8SResourcesList(objects)
+}

coordinator/kubernetes/k8sapi/resources/configmaps_test.go

@@ -0,0 +1,49 @@
+package resources
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	k8s "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestConfigMaps(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	configMaps := ConfigMaps{
+		&k8s.ConfigMap{
+			TypeMeta: v1.TypeMeta{
+				Kind:       "ConfigMap",
+				APIVersion: "v1",
+			},
+			Data: map[string]string{"key": "value1"},
+		},
+		&k8s.ConfigMap{
+			TypeMeta: v1.TypeMeta{
+				Kind:       "ConfigMap",
+				APIVersion: "v1",
+			},
+			Data: map[string]string{"key": "value2"},
+		},
+	}
+	data, err := configMaps.Marshal()
+	require.NoError(err)
+
+	assert.Equal(`apiVersion: v1
+data:
+  key: value1
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+---
+apiVersion: v1
+data:
+  key: value2
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+`, string(data))
+}

coordinator/kubernetes/k8sapi/resources/secrets.go

@@ -0,0 +1,18 @@
+package resources
+
+import (
+	k8s "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// ConfigMaps represent a list of k8s Secret.
+type Secrets []*k8s.Secret
+
+// Marshal marshals secrets into multiple YAML documents.
+func (s Secrets) Marshal() ([]byte, error) {
+	objects := make([]runtime.Object, len(s))
+	for i := range s {
+		objects[i] = s[i]
+	}
+	return MarshalK8SResourcesList(objects)
+}

coordinator/kubernetes/k8sapi/resources/secrets_test.go

@@ -0,0 +1,49 @@
+package resources
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	k8s "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestSecrets(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	secrets := Secrets{
+		&k8s.Secret{
+			TypeMeta: v1.TypeMeta{
+				Kind:       "Secret",
+				APIVersion: "v1",
+			},
+			Data: map[string][]byte{"key": []byte("value1")},
+		},
+		&k8s.Secret{
+			TypeMeta: v1.TypeMeta{
+				Kind:       "Secret",
+				APIVersion: "v1",
+			},
+			Data: map[string][]byte{"key": []byte("value2")},
+		},
+	}
+	data, err := secrets.Marshal()
+	require.NoError(err)
+
+	assert.Equal(`apiVersion: v1
+data:
+  key: dmFsdWUx
+kind: Secret
+metadata:
+  creationTimestamp: null
+---
+apiVersion: v1
+data:
+  key: dmFsdWUy
+kind: Secret
+metadata:
+  creationTimestamp: null
+`, string(data))
+}

coordinator/kubernetes/k8sapi/util.go

@@ -26,7 +26,7 @@ type ClusterUtil interface {
 	JoinCluster(joinConfig []byte) error
 	SetupPodNetwork(kubectl Client, podNetworkConfiguration resources.Marshaler) error
 	SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler) error
-	SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler) error
+	SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error
 	RestartKubelet() error
 }
 
@@ -96,12 +96,24 @@ func (k *KubernetesUtil) SetupPodNetwork(kubectl Client, podNetworkConfiguration
 	return exec.Command("kubectl", "--kubeconfig", kubeConfig, "-n", "kube-system", "patch", "deployment", "coredns", "--type", "json", "-p", "[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.kubernetes.io/network-unavailable\",\"value\":\"\",\"effect\":\"NoSchedule\"}}]").Run()
 }
 
+// SetupAutoscaling deploys the k8s cluster autoscaler.
 func (k *KubernetesUtil) SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler) error {
 	return kubectl.Apply(clusterAutoscalerConfiguration, true)
 }
 
-func (k *KubernetesUtil) SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler) error {
+// SetupCloudControllerManager deploys the k8s cloud-controller-manager.
-	return kubectl.Apply(cloudControllerManagerConfiguration, true)
+func (k *KubernetesUtil) SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error {
+	if err := kubectl.Apply(configMaps, true); err != nil {
+		return fmt.Errorf("applying ccm ConfigMaps failed: %w", err)
+	}
+	if err := kubectl.Apply(secrets, true); err != nil {
+		return fmt.Errorf("applying ccm Secrets failed: %w", err)
+	}
+	if err := kubectl.Apply(cloudControllerManagerConfiguration, true); err != nil {
+		return fmt.Errorf("applying ccm failed: %w", err)
+	}
+	return nil
+}
 }
 
 // JoinCluster joins existing kubernetes cluster using kubeadm join.

coordinator/kubernetes/kubernetes.go

@@ -72,16 +72,21 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
 		return nil, fmt.Errorf("setup of pod network failed: %w", err)
 	}
 
+	if in.SupportsCloudControllerManager {
+		cloudControllerManagerConfiguration := resources.NewDefaultCloudControllerManagerDeployment(
+			in.CloudControllerManagerName, in.CloudControllerManagerImage, in.CloudControllerManagerPath, in.CloudControllerManagerExtraArgs,
+			in.CloudControllerManagerVolumes, in.CloudControllerManagerVolumeMounts, in.CloudControllerManagerEnv,
+		)
+		if err := k.clusterUtil.SetupCloudControllerManager(k.client, cloudControllerManagerConfiguration, in.CloudControllerManagerConfigMaps, in.CloudControllerManagerSecrets); err != nil {
+			return nil, fmt.Errorf("failed to setup cloud-controller-manager: %w", err)
+		}
+	}
 	if in.SupportClusterAutoscaler {
 		clusterAutoscalerConfiguration := resources.NewDefaultAutoscalerDeployment()
 		clusterAutoscalerConfiguration.SetAutoscalerCommand(in.AutoscalingCloudprovider, in.AutoscalingNodeGroups)
 		if err := k.clusterUtil.SetupAutoscaling(k.client, clusterAutoscalerConfiguration); err != nil {
 			return nil, fmt.Errorf("failed to setup cluster-autoscaler: %w", err)
 		}
-		cloudControllerManagerConfiguration := resources.NewDefaultCloudControllerManagerDeployment(in.CloudControllerManagerName, in.CloudControllerManagerImage, in.CloudControllerManagerPath)
-		if err := k.clusterUtil.SetupCloudControllerManager(k.client, cloudControllerManagerConfiguration); err != nil {
-			return nil, fmt.Errorf("failed to setup cloud-controller-manager: %w", err)
-		}
 	}
 
 	return joinK8SClusterRequest, nil

coordinator/kubernetes/kubernetes_test.go

@@ -25,7 +25,7 @@ type stubClusterUtil struct {
 	initClusterErr                   error
 	setupPodNetworkErr               error
 	setupAutoscalingError            error
-	SetupCloudControllerManagerError error
+	setupCloudControllerManagerError error
 	joinClusterErr                   error
 	restartKubeletErr                error
 
@@ -46,8 +46,9 @@ func (s *stubClusterUtil) SetupAutoscaling(kubectl k8sapi.Client, clusterAutosca
 	return s.setupAutoscalingError
 }
 
-func (s *stubClusterUtil) SetupCloudControllerManager(kubectl k8sapi.Client, cloudControllerManagerConfiguration resources.Marshaler) error {
+func (s *stubClusterUtil) SetupCloudControllerManager(kubectl k8sapi.Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error {
-	return s.SetupCloudControllerManagerError
+	return s.setupCloudControllerManagerError
+}
 }
 
 func (s *stubClusterUtil) JoinCluster(joinConfig []byte) error {

coordinator/pubapi/coord.go

@@ -66,7 +66,7 @@ func (a *API) ActivateAsCoordinator(in *pubproto.ActivateAsCoordinatorRequest, s
 		return status.Errorf(codes.Internal, "%v", err)
 	}
 
-	kubeconfig, err := a.core.InitCluster(in.AutoscalingNodeGroups)
+	kubeconfig, err := a.core.InitCluster(in.AutoscalingNodeGroups, in.CloudServiceAccountUri)
 	if err != nil {
 		return status.Errorf(codes.Internal, "%v", err)
 	}

coordinator/pubapi/core.go

@@ -26,6 +26,6 @@ type Core interface {
 	AddPeer(peer.Peer) error
 	UpdatePeers([]peer.Peer) error
 
-	InitCluster(autoscalingNodeGroups []string) ([]byte, error)
+	InitCluster(autoscalingNodeGroups []string, cloudServiceAccountURI string) ([]byte, error)
 	JoinCluster(kubeadm.BootstrapTokenDiscovery) error
 }

coordinator/pubapi/core_test.go

@@ -92,7 +92,7 @@ func (c *fakeCore) UpdatePeers(peers []peer.Peer) error {
 	return nil
 }
 
-func (c *fakeCore) InitCluster(autoscalingNodeGroups []string) ([]byte, error) {
+func (c *fakeCore) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountURI string) ([]byte, error) {
 	c.autoscalingNodeGroups = autoscalingNodeGroups
 	return c.kubeconfig, nil
 }