Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-02-22 16:00:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

Cloud providers: Extend CCM with ExtraArgs / ConfigMaps / Secrets / Volumes / VolumeMounts and provide CloudServiceAccountURI

Browse Source
This commit is contained in:
Malte Poll 2022-03-25 10:42:27 +01:00 committed by Malte Poll
parent bf726ebd87
commit 2158377f9f
17 changed files with 366 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
coordinator/core/cloud_provider.go
View File

@ -6,7 +6,9 @@ import (
"fmt"
"net"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/coordinator/role"
k8s "k8s.io/api/core/v1"
)
var ErrUnimplemented = errors.New("unimplemented")
@ -52,6 +54,21 @@ type CloudControllerManager interface {
Path() string
// Name returns the cloud-provider name as used by k8s cloud-controller-manager (k8s.gcr.io/cloud-controller-manager).
Name() string
// ExtraArgs returns a list of arguments to append to the cloud-controller-manager command.
ExtraArgs() []string
// ConfigMaps returns a list of ConfigMaps to deploy together with the k8s cloud-controller-manager
// Reference: https://kubernetes.io/docs/concepts/configuration/configmap/ .
ConfigMaps(instance Instance) (resources.ConfigMaps, error)
// Secrets returns a list of secrets to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/configuration/secret/ .
Secrets(instance Instance, cloudServiceAccountURI string) (resources.Secrets, error)
// Volumes returns a list of volumes to deploy together with the k8s cloud-controller-manager.
// Reference: https://kubernetes.io/docs/concepts/storage/volumes/ .
Volumes() []k8s.Volume
// VolumeMounts a list of of volume mounts to deploy together with the k8s cloud-controller-manager.
VolumeMounts() []k8s.VolumeMount
// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cloud-controller-manager.
Env() []k8s.EnvVar
// PrepareInstance is called on every instance before deploying the cloud-controller-manager.
// Allows for cloud-provider specific hooks.
PrepareInstance(instance Instance, vpnIP string) error
@ -153,6 +170,30 @@ func (f *CloudControllerManagerFake) Name() string {
return "fake"
}
func (f *CloudControllerManagerFake) ExtraArgs() []string {
return []string{}
}
func (f *CloudControllerManagerFake) ConfigMaps(instance Instance) (resources.ConfigMaps, error) {
return []*k8s.ConfigMap{}, nil
}
func (f *CloudControllerManagerFake) Secrets(instance Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
return []*k8s.Secret{}, nil
}
func (f *CloudControllerManagerFake) Volumes() []k8s.Volume {
return []k8s.Volume{}
}
func (f *CloudControllerManagerFake) VolumeMounts() []k8s.VolumeMount {
return []k8s.VolumeMount{}
}
func (f *CloudControllerManagerFake) Env() []k8s.EnvVar {
return []k8s.EnvVar{}
}
func (f *CloudControllerManagerFake) PrepareInstance(instance Instance, vpnIP string) error {
return nil
}

50
coordinator/core/cluster.go
View File

@ -5,6 +5,7 @@ import (
"strings"
"github.com/edgelesssys/constellation/coordinator/kubernetes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/coordinator/role"
"go.uber.org/zap"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
@ -16,12 +17,17 @@ func (c *Core) GetK8sJoinArgs() (*kubeadm.BootstrapTokenDiscovery, error) {
}
// InitCluster initializes the cluster, stores the join args, and returns the kubeconfig.
func (c *Core) InitCluster(autoscalingNodeGroups []string) ([]byte, error) {
func (c *Core) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountURI string) ([]byte, error) {
var nodeName string
var providerID string
var instance Instance
var ccmConfigMaps resources.ConfigMaps
var ccmSecrets resources.Secrets
var err error
nodeIP := coordinatorVPNIP.String()
var err error
if c.metadata.Supported() {
var err error
instance, err := c.metadata.Self(context.TODO())
instance, err = c.metadata.Self(context.TODO())
if err != nil {
c.zaplogger.Error("Retrieving own instance metadata failed", zap.Error(err))
return nil, err
@ -33,25 +39,41 @@ func (c *Core) InitCluster(autoscalingNodeGroups []string) ([]byte, error) {
}
if c.cloudControllerManager.Supported() && c.metadata.Supported() {
c.zaplogger.Info("Preparing node for cloud-controller-manager")
if err := PrepareInstanceForCCM(context.TODO(), c.metadata, c.cloudControllerManager, coordinatorVPNIP.String()); err != nil {
c.zaplogger.Error("Preparing node for CCM failed", zap.Error(err))
return nil, err
}
ccmConfigMaps, err = c.cloudControllerManager.ConfigMaps(instance)
if err != nil {
c.zaplogger.Error("Defining ConfigMaps for CCM failed", zap.Error(err))
return nil, err
}
ccmSecrets, err = c.cloudControllerManager.Secrets(instance, cloudServiceAccountURI)
if err != nil {
c.zaplogger.Error("Defining Secrets for CCM failed", zap.Error(err))
return nil, err
}
}
c.zaplogger.Info("Initializing cluster")
joinCommand, err := c.kube.InitCluster(kubernetes.InitClusterInput{
APIServerAdvertiseIP: coordinatorVPNIP.String(),
NodeName: k8sCompliantHostname(nodeName),
ProviderID: providerID,
SupportClusterAutoscaler: c.clusterAutoscaler.Supported(),
AutoscalingCloudprovider: c.clusterAutoscaler.Name(),
AutoscalingNodeGroups: autoscalingNodeGroups,
SupportsCloudControllerManager: c.cloudControllerManager.Supported(),
CloudControllerManagerName: c.cloudControllerManager.Name(),
CloudControllerManagerImage: c.cloudControllerManager.Image(),
CloudControllerManagerPath: c.cloudControllerManager.Path(),
APIServerAdvertiseIP: coordinatorVPNIP.String(),
NodeIP: nodeIP,
NodeName: k8sCompliantHostname(nodeName),
ProviderID: providerID,
SupportClusterAutoscaler: c.clusterAutoscaler.Supported(),
AutoscalingCloudprovider: c.clusterAutoscaler.Name(),
AutoscalingNodeGroups: autoscalingNodeGroups,
SupportsCloudControllerManager: c.cloudControllerManager.Supported(),
CloudControllerManagerName: c.cloudControllerManager.Name(),
CloudControllerManagerImage: c.cloudControllerManager.Image(),
CloudControllerManagerPath: c.cloudControllerManager.Path(),
CloudControllerManagerExtraArgs: c.cloudControllerManager.ExtraArgs(),
CloudControllerManagerConfigMaps: ccmConfigMaps,
CloudControllerManagerSecrets: ccmSecrets,
CloudControllerManagerVolumes: c.cloudControllerManager.Volumes(),
CloudControllerManagerVolumeMounts: c.cloudControllerManager.VolumeMounts(),
CloudControllerManagerEnv: c.cloudControllerManager.Env(),
})
if err != nil {
c.zaplogger.Error("Initializing cluster failed", zap.Error(err))

40
coordinator/core/cluster_test.go
View File

@ -7,9 +7,11 @@ import (
"github.com/edgelesssys/constellation/coordinator/attestation/vtpm"
"github.com/edgelesssys/constellation/coordinator/kubernetes"
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
k8s "k8s.io/api/core/v1"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
@ -36,6 +38,7 @@ func TestInitCluster(t *testing.T) {
SupportsCloudControllerManager: false,
SupportClusterAutoscaler: false,
AutoscalingNodeGroups: []string{"someNodeGroup"},
CloudControllerManagerEnv: []k8s.EnvVar{},
},
expectErr: false,
},
@ -56,6 +59,7 @@ func TestInitCluster(t *testing.T) {
ProviderID: "fake://providerid",
SupportsCloudControllerManager: false,
SupportClusterAutoscaler: false,
CloudControllerManagerEnv: []k8s.EnvVar{},
},
expectErr: false,
},
@ -85,6 +89,7 @@ func TestInitCluster(t *testing.T) {
SupportClusterAutoscaler: true,
AutoscalingCloudprovider: "some-name",
AutoscalingNodeGroups: []string{"someNodeGroup"},
CloudControllerManagerEnv: []k8s.EnvVar{},
},
expectErr: false,
},
@ -106,6 +111,7 @@ func TestInitCluster(t *testing.T) {
CloudControllerManagerName: "some-name",
CloudControllerManagerImage: "someImage",
CloudControllerManagerPath: "/some/path",
CloudControllerManagerEnv: []k8s.EnvVar{},
},
expectErr: false,
},
@ -136,6 +142,7 @@ func TestInitCluster(t *testing.T) {
expectErr: false,
expectedInitClusterInput: kubernetes.InitClusterInput{
APIServerAdvertiseIP: "10.118.0.1",
CloudControllerManagerEnv: []k8s.EnvVar{},
},
},
"getting kubeconfig fail detected": {
@ -162,7 +169,7 @@ func TestInitCluster(t *testing.T) {
core, err := NewCore(&stubVPN{}, &tc.cluster, &tc.metadata, &tc.cloudControllerManager, &tc.clusterAutoscaler, zapLogger, vtpm.OpenSimulatedTPM, nil)
require.NoError(err)
kubeconfig, err := core.InitCluster(tc.autoscalingNodeGroups)
kubeconfig, err := core.InitCluster(tc.autoscalingNodeGroups, "cloud-service-account-uri")
if tc.expectErr {
assert.Error(err)
@ -368,6 +375,13 @@ type stubCloudControllerManager struct {
pathRes string
nameRes string
prepareInstanceRes error
extraArgsRes []string
configMapsRes resources.ConfigMaps
configMapsErr error
secretsRes resources.Secrets
secretsErr error
volumesRes []k8s.Volume
volumeMountRes []k8s.VolumeMount
supportedRes bool
prepareInstanceRequests []prepareInstanceRequest
@ -393,6 +407,30 @@ func (s *stubCloudControllerManager) PrepareInstance(instance Instance, vpnIP st
return s.prepareInstanceRes
}
func (s *stubCloudControllerManager) ExtraArgs() []string {
return s.extraArgsRes
}
func (s *stubCloudControllerManager) ConfigMaps(instance Instance) (resources.ConfigMaps, error) {
return s.configMapsRes, s.configMapsErr
}
func (s *stubCloudControllerManager) Secrets(instance Instance, cloudServiceAccountURI string) (resources.Secrets, error) {
return s.secretsRes, s.secretsErr
}
func (s *stubCloudControllerManager) Volumes() []k8s.Volume {
return s.volumesRes
}
func (s *stubCloudControllerManager) VolumeMounts() []k8s.VolumeMount {
return s.volumeMountRes
}
func (s *stubCloudControllerManager) Env() []k8s.EnvVar {
return []k8s.EnvVar{}
}
func (s *stubCloudControllerManager) Supported() bool {
return s.supportedRes
}

32
coordinator/kubernetes/cluster_input.go
View File

@ -1,15 +1,27 @@
package kubernetes
import (
"github.com/edgelesssys/constellation/coordinator/kubernetes/k8sapi/resources"
k8s "k8s.io/api/core/v1"
)
// InitClusterInput collects the arguments to initialize a new cluster.
type InitClusterInput struct {
APIServerAdvertiseIP string
NodeName string
ProviderID string
SupportClusterAutoscaler bool
AutoscalingCloudprovider string
AutoscalingNodeGroups []string
SupportsCloudControllerManager bool
CloudControllerManagerName string
CloudControllerManagerImage string
CloudControllerManagerPath string
APIServerAdvertiseIP string
NodeIP string
NodeName string
ProviderID string
SupportClusterAutoscaler bool
AutoscalingCloudprovider string
AutoscalingNodeGroups []string
SupportsCloudControllerManager bool
CloudControllerManagerName string
CloudControllerManagerImage string
CloudControllerManagerPath string
CloudControllerManagerExtraArgs []string
CloudControllerManagerConfigMaps resources.ConfigMaps
CloudControllerManagerSecrets resources.Secrets
CloudControllerManagerVolumes []k8s.Volume
CloudControllerManagerVolumeMounts []k8s.VolumeMount
CloudControllerManagerEnv []k8s.EnvVar
}

2
coordinator/kubernetes/k8sapi/kubectl/client/client_test.go
View File

@ -250,7 +250,7 @@ func TestGetObjects(t *testing.T) {
expectErr: false,
},
"GetObjects works on cloud-controller-manager deployment": {
expectedResources: resources.NewDefaultCloudControllerManagerDeployment("someProvider", "someImage", "somePath"),
expectedResources: resources.NewDefaultCloudControllerManagerDeployment("someProvider", "someImage", "somePath", nil, nil, nil, nil),
resourcesYAML: string(nginxDeplYAML),
expectErr: false,
},

122
coordinator/kubernetes/k8sapi/resources/cloud_controller_manager.go
View File

@ -9,6 +9,8 @@ import (
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const defaultCIDR = "10.244.0.0/16"
type cloudControllerManagerDeployment struct {
ServiceAccount k8s.ServiceAccount
ClusterRoleBinding rbac.ClusterRoleBinding
@ -19,7 +21,58 @@ type cloudControllerManagerDeployment struct {
// https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/admin/cloud/ccm-example.yaml
// https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/#cloud-controller-manager
func NewDefaultCloudControllerManagerDeployment(cloudProvider string, image string, path string) *cloudControllerManagerDeployment {
// NewDefaultCloudControllerManagerDeployment creates a new *cloudControllerManagerDeployment, customized for the CSP.
func NewDefaultCloudControllerManagerDeployment(cloudProvider, image, path string, extraArgs []string, extraVolumes []k8s.Volume, extraVolumeMounts []k8s.VolumeMount, env []k8s.EnvVar) *cloudControllerManagerDeployment {
command := []string{
path,
fmt.Sprintf("--cloud-provider=%s", cloudProvider),
"--leader-elect=true",
"--allocate-node-cidrs=false",
"--configure-cloud-routes=false",
fmt.Sprintf("--cluster-cidr=%s", defaultCIDR),
"-v=2",
}
command = append(command, extraArgs...)
volumes := []k8s.Volume{
{
Name: "etckubernetes",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{Path: "/etc/kubernetes"},
},
},
{
Name: "etcssl",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{Path: "/etc/ssl"},
},
},
{
Name: "etcpki",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{Path: "/etc/pki"},
},
},
}
volumes = append(volumes, extraVolumes...)
volumeMounts := []k8s.VolumeMount{
{
MountPath: "/etc/kubernetes",
Name: "etckubernetes",
ReadOnly: true,
},
{
MountPath: "/etc/ssl",
Name: "etcssl",
ReadOnly: true,
},
{
MountPath: "/etc/pki",
Name: "etcpki",
ReadOnly: true,
},
}
volumeMounts = append(volumeMounts, extraVolumeMounts...)
return &cloudControllerManagerDeployment{
ServiceAccount: k8s.ServiceAccount{
TypeMeta: meta.TypeMeta{
@ -80,69 +133,14 @@ func NewDefaultCloudControllerManagerDeployment(cloudProvider string, image stri
ServiceAccountName: "cloud-controller-manager",
Containers: []k8s.Container{
{
Name: "cloud-controller-manager",
Image: image,
Command: []string{
path,
fmt.Sprintf("--cloud-provider=%s", cloudProvider),
"--leader-elect=true",
"--allocate-node-cidrs=false",
"--configure-cloud-routes=false",
"--controllers=cloud-node,cloud-node-lifecycle",
"--use-service-account-credentials",
"--cluster-cidr=10.244.0.0/16",
"-v=2",
},
VolumeMounts: []k8s.VolumeMount{
{
MountPath: "/etc/kubernetes",
Name: "etckubernetes",
ReadOnly: true,
},
{
MountPath: "/etc/ssl",
Name: "etcssl",
ReadOnly: true,
},
{
MountPath: "/etc/pki",
Name: "etcpki",
ReadOnly: true,
},
{
MountPath: "/etc/gce.conf",
Name: "gceconf",
ReadOnly: true,
},
},
},
},
Volumes: []k8s.Volume{
{
Name: "etckubernetes",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{Path: "/etc/kubernetes"},
},
},
{
Name: "etcssl",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{Path: "/etc/ssl"},
},
},
{
Name: "etcpki",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{Path: "/etc/pki"},
},
},
{
Name: "gceconf",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{Path: "/etc/gce.conf"},
},
Name: "cloud-controller-manager",
Image: image,
Command: command,
VolumeMounts: volumeMounts,
Env: env,
},
},
Volumes: volumes,
Tolerations: []k8s.Toleration{
{
Key: "node.cloudprovider.kubernetes.io/uninitialized",

3
coordinator/kubernetes/k8sapi/resources/cloud_controller_manager_test.go
View File

@ -5,13 +5,14 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
k8s "k8s.io/api/core/v1"
)
func TestCloudControllerMarshalUnmarshal(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
cloudControllerManagerDepl := NewDefaultCloudControllerManagerDeployment("dummy-cloudprovider", "some-image:latest", "/dummy_path")
cloudControllerManagerDepl := NewDefaultCloudControllerManagerDeployment("dummy-cloudprovider", "some-image:latest", "/dummy_path", []string{}, []k8s.Volume{}, []k8s.VolumeMount{}, nil)
data, err := cloudControllerManagerDepl.Marshal()
require.NoError(err)

18
coordinator/kubernetes/k8sapi/resources/configmaps.go Normal file
View File

@ -0,0 +1,18 @@
package resources
import (
k8s "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
)
// ConfigMaps represent a list of k8s ConfigMap.
type ConfigMaps []*k8s.ConfigMap
// Marshal marshals config maps into multiple YAML documents.
func (s ConfigMaps) Marshal() ([]byte, error) {
objects := make([]runtime.Object, len(s))
for i := range s {
objects[i] = s[i]
}
return MarshalK8SResourcesList(objects)
}

49
coordinator/kubernetes/k8sapi/resources/configmaps_test.go Normal file
View File

@ -0,0 +1,49 @@
package resources
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
k8s "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestConfigMaps(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
configMaps := ConfigMaps{
&k8s.ConfigMap{
TypeMeta: v1.TypeMeta{
Kind: "ConfigMap",
APIVersion: "v1",
},
Data: map[string]string{"key": "value1"},
},
&k8s.ConfigMap{
TypeMeta: v1.TypeMeta{
Kind: "ConfigMap",
APIVersion: "v1",
},
Data: map[string]string{"key": "value2"},
},
}
data, err := configMaps.Marshal()
require.NoError(err)
assert.Equal(`apiVersion: v1
data:
key: value1
kind: ConfigMap
metadata:
creationTimestamp: null
---
apiVersion: v1
data:
key: value2
kind: ConfigMap
metadata:
creationTimestamp: null
`, string(data))
}

18
coordinator/kubernetes/k8sapi/resources/secrets.go Normal file
View File

@ -0,0 +1,18 @@
package resources
import (
k8s "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
)
// ConfigMaps represent a list of k8s Secret.
type Secrets []*k8s.Secret
// Marshal marshals secrets into multiple YAML documents.
func (s Secrets) Marshal() ([]byte, error) {
objects := make([]runtime.Object, len(s))
for i := range s {
objects[i] = s[i]
}
return MarshalK8SResourcesList(objects)
}

49
coordinator/kubernetes/k8sapi/resources/secrets_test.go Normal file
View File

@ -0,0 +1,49 @@
package resources
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
k8s "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestSecrets(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
secrets := Secrets{
&k8s.Secret{
TypeMeta: v1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
Data: map[string][]byte{"key": []byte("value1")},
},
&k8s.Secret{
TypeMeta: v1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
Data: map[string][]byte{"key": []byte("value2")},
},
}
data, err := secrets.Marshal()
require.NoError(err)
assert.Equal(`apiVersion: v1
data:
key: dmFsdWUx
kind: Secret
metadata:
creationTimestamp: null
---
apiVersion: v1
data:
key: dmFsdWUy
kind: Secret
metadata:
creationTimestamp: null
`, string(data))
}

18
coordinator/kubernetes/k8sapi/util.go
View File

@ -26,7 +26,7 @@ type ClusterUtil interface {
JoinCluster(joinConfig []byte) error
SetupPodNetwork(kubectl Client, podNetworkConfiguration resources.Marshaler) error
SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler) error
SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler) error
SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error
RestartKubelet() error
}
@ -96,12 +96,24 @@ func (k *KubernetesUtil) SetupPodNetwork(kubectl Client, podNetworkConfiguration
return exec.Command("kubectl", "--kubeconfig", kubeConfig, "-n", "kube-system", "patch", "deployment", "coredns", "--type", "json", "-p", "[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.kubernetes.io/network-unavailable\",\"value\":\"\",\"effect\":\"NoSchedule\"}}]").Run()
}
// SetupAutoscaling deploys the k8s cluster autoscaler.
func (k *KubernetesUtil) SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration resources.Marshaler) error {
return kubectl.Apply(clusterAutoscalerConfiguration, true)
}
func (k *KubernetesUtil) SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler) error {
return kubectl.Apply(cloudControllerManagerConfiguration, true)
// SetupCloudControllerManager deploys the k8s cloud-controller-manager.
func (k *KubernetesUtil) SetupCloudControllerManager(kubectl Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error {
if err := kubectl.Apply(configMaps, true); err != nil {
return fmt.Errorf("applying ccm ConfigMaps failed: %w", err)
}
if err := kubectl.Apply(secrets, true); err != nil {
return fmt.Errorf("applying ccm Secrets failed: %w", err)
}
if err := kubectl.Apply(cloudControllerManagerConfiguration, true); err != nil {
return fmt.Errorf("applying ccm failed: %w", err)
}
return nil
}
}
// JoinCluster joins existing kubernetes cluster using kubeadm join.

13
coordinator/kubernetes/kubernetes.go
View File

@ -72,16 +72,21 @@ func (k *KubeWrapper) InitCluster(in InitClusterInput) (*kubeadm.BootstrapTokenD
return nil, fmt.Errorf("setup of pod network failed: %w", err)
}
if in.SupportsCloudControllerManager {
cloudControllerManagerConfiguration := resources.NewDefaultCloudControllerManagerDeployment(
in.CloudControllerManagerName, in.CloudControllerManagerImage, in.CloudControllerManagerPath, in.CloudControllerManagerExtraArgs,
in.CloudControllerManagerVolumes, in.CloudControllerManagerVolumeMounts, in.CloudControllerManagerEnv,
)
if err := k.clusterUtil.SetupCloudControllerManager(k.client, cloudControllerManagerConfiguration, in.CloudControllerManagerConfigMaps, in.CloudControllerManagerSecrets); err != nil {
return nil, fmt.Errorf("failed to setup cloud-controller-manager: %w", err)
}
}
if in.SupportClusterAutoscaler {
clusterAutoscalerConfiguration := resources.NewDefaultAutoscalerDeployment()
clusterAutoscalerConfiguration.SetAutoscalerCommand(in.AutoscalingCloudprovider, in.AutoscalingNodeGroups)
if err := k.clusterUtil.SetupAutoscaling(k.client, clusterAutoscalerConfiguration); err != nil {
return nil, fmt.Errorf("failed to setup cluster-autoscaler: %w", err)
}
cloudControllerManagerConfiguration := resources.NewDefaultCloudControllerManagerDeployment(in.CloudControllerManagerName, in.CloudControllerManagerImage, in.CloudControllerManagerPath)
if err := k.clusterUtil.SetupCloudControllerManager(k.client, cloudControllerManagerConfiguration); err != nil {
return nil, fmt.Errorf("failed to setup cloud-controller-manager: %w", err)
}
}
return joinK8SClusterRequest, nil

7
coordinator/kubernetes/kubernetes_test.go
View File

@ -25,7 +25,7 @@ type stubClusterUtil struct {
initClusterErr error
setupPodNetworkErr error
setupAutoscalingError error
SetupCloudControllerManagerError error
setupCloudControllerManagerError error
joinClusterErr error
restartKubeletErr error
@ -46,8 +46,9 @@ func (s *stubClusterUtil) SetupAutoscaling(kubectl k8sapi.Client, clusterAutosca
return s.setupAutoscalingError
}
func (s *stubClusterUtil) SetupCloudControllerManager(kubectl k8sapi.Client, cloudControllerManagerConfiguration resources.Marshaler) error {
return s.SetupCloudControllerManagerError
func (s *stubClusterUtil) SetupCloudControllerManager(kubectl k8sapi.Client, cloudControllerManagerConfiguration resources.Marshaler, configMaps resources.Marshaler, secrets resources.Marshaler) error {
return s.setupCloudControllerManagerError
}
}
func (s *stubClusterUtil) JoinCluster(joinConfig []byte) error {

2
coordinator/pubapi/coord.go
View File

@ -66,7 +66,7 @@ func (a *API) ActivateAsCoordinator(in *pubproto.ActivateAsCoordinatorRequest, s
return status.Errorf(codes.Internal, "%v", err)
}
kubeconfig, err := a.core.InitCluster(in.AutoscalingNodeGroups)
kubeconfig, err := a.core.InitCluster(in.AutoscalingNodeGroups, in.CloudServiceAccountUri)
if err != nil {
return status.Errorf(codes.Internal, "%v", err)
}

2
coordinator/pubapi/core.go
View File

@ -26,6 +26,6 @@ type Core interface {
AddPeer(peer.Peer) error
UpdatePeers([]peer.Peer) error
InitCluster(autoscalingNodeGroups []string) ([]byte, error)
InitCluster(autoscalingNodeGroups []string, cloudServiceAccountURI string) ([]byte, error)
JoinCluster(kubeadm.BootstrapTokenDiscovery) error
}

2
coordinator/pubapi/core_test.go
View File

@ -92,7 +92,7 @@ func (c *fakeCore) UpdatePeers(peers []peer.Peer) error {
return nil
}
func (c *fakeCore) InitCluster(autoscalingNodeGroups []string) ([]byte, error) {
func (c *fakeCore) InitCluster(autoscalingNodeGroups []string, cloudServiceAccountURI string) ([]byte, error) {
c.autoscalingNodeGroups = autoscalingNodeGroups
return c.kubeconfig, nil
}