AB#2538: deploy CCM via Helm

Also move helmloader interface/stubs

CMakeLists.txt

@@ -63,7 +63,9 @@ add_test(NAME integration-node-operator COMMAND make test WORKING_DIRECTORY ${CM
 add_test(NAME integration-csi COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/csi)
 add_test(NAME integration-dm COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/disk-mapper/internal)
 add_test(NAME integration-license COMMAND bash -c "go test -tags integration" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/internal/license)
-add_test(NAME helm-lint COMMAND bash -c "helm lint * --set kms.image='ghcr.io/edgelesssys/constellation/kms:latest' --set join-service.csp='QEMU' \
+add_test(NAME helm-lint COMMAND bash -c "helm lint * --set tags.GCP=true --set kms.image='ghcr.io/edgelesssys/constellation/kms:latest' --set join-service.csp='GCP' \
   --set join-service.enforcedPCRs='[]' --set join-service.image='ghcr.io/edgelesssys/constellation/join-service:latest' --set join-service.measurements='[]' \
-  --set join-service.measurementSalt='deadbeef' --set kms.salt='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' --set kms.masterSecret='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'"
+  --set join-service.measurementSalt='deadbeef' --set kms.salt='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' --set kms.masterSecret='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \
+  --set ccm.csp='GCP' --set ccm.subnetworkCIDR='192.0.2.0/24' --set ccm.GCP.image='ghcr.io/edgelesssys/cloud-provider-gcp:latest' --set ccm.GCP.projectID='demoproject-581925' \
+  --set ccm.GCP.uid='foobar' --set ccm.GCP.secretData='jsonstring'"
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/cli/internal/helm/charts/edgeless/)

bootstrapper/internal/kubernetes/k8sapi/constants.go

@@ -14,5 +14,4 @@ const (
 	kubeletPath        = "/run/state/bin/kubelet"
 	kubeletServicePath = "/usr/lib/systemd/system/kubelet.service"
 	executablePerm     = 0o544
-	systemdUnitPerm    = 0o644
 )

bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go

@@ -280,11 +280,6 @@ func TestGetObjects(t *testing.T) {
 			resourcesYAML: string(nginxDeplYAML),
 			wantErr:       false,
 		},
-		"GetObjects works on cloud-controller-manager deployment": {
-			wantResources: resources.NewDefaultCloudControllerManagerDeployment("someProvider", "someImage", "somePath", "someCIDR", nil, nil, nil, nil),
-			resourcesYAML: string(nginxDeplYAML),
-			wantErr:       false,
-		},
 		"GetObjects Marshal failure detected": {
 			wantResources: &unmarshableResource{},
 			resourcesYAML: string(nginxDeplYAML),

bootstrapper/internal/kubernetes/k8sapi/resources/cloud_controller_manager.go

@@ -1,179 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-package resources
-
-import (
-	"fmt"
-
-	"github.com/edgelesssys/constellation/v2/internal/kubernetes"
-	apps "k8s.io/api/apps/v1"
-	k8s "k8s.io/api/core/v1"
-	rbac "k8s.io/api/rbac/v1"
-	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-type CloudControllerManagerDeployment struct {
-	ServiceAccount     k8s.ServiceAccount
-	ClusterRoleBinding rbac.ClusterRoleBinding
-	DaemonSet          apps.DaemonSet
-}
-
-// references:
-// https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/admin/cloud/ccm-example.yaml
-// https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/#cloud-controller-manager
-
-// NewDefaultCloudControllerManagerDeployment creates a new *cloudControllerManagerDeployment, customized for the CSP.
-func NewDefaultCloudControllerManagerDeployment(cloudProvider, image, path, podCIDR string, extraArgs []string, extraVolumes []k8s.Volume, extraVolumeMounts []k8s.VolumeMount, env []k8s.EnvVar) *CloudControllerManagerDeployment {
-	command := []string{
-		path,
-		fmt.Sprintf("--cloud-provider=%s", cloudProvider),
-		"--leader-elect=true",
-		fmt.Sprintf("--cluster-cidr=%s", podCIDR),
-		"-v=2",
-	}
-	command = append(command, extraArgs...)
-	volumes := []k8s.Volume{
-		{
-			Name: "etckubernetes",
-			VolumeSource: k8s.VolumeSource{
-				HostPath: &k8s.HostPathVolumeSource{Path: "/etc/kubernetes"},
-			},
-		},
-		{
-			Name: "etcssl",
-			VolumeSource: k8s.VolumeSource{
-				HostPath: &k8s.HostPathVolumeSource{Path: "/etc/ssl"},
-			},
-		},
-		{
-			Name: "etcpki",
-			VolumeSource: k8s.VolumeSource{
-				HostPath: &k8s.HostPathVolumeSource{Path: "/etc/pki"},
-			},
-		},
-	}
-	volumes = append(volumes, extraVolumes...)
-	volumeMounts := []k8s.VolumeMount{
-		{
-			MountPath: "/etc/kubernetes",
-			Name:      "etckubernetes",
-			ReadOnly:  true,
-		},
-		{
-			MountPath: "/etc/ssl",
-			Name:      "etcssl",
-			ReadOnly:  true,
-		},
-		{
-			MountPath: "/etc/pki",
-			Name:      "etcpki",
-			ReadOnly:  true,
-		},
-	}
-	volumeMounts = append(volumeMounts, extraVolumeMounts...)
-
-	return &CloudControllerManagerDeployment{
-		ServiceAccount: k8s.ServiceAccount{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "v1",
-				Kind:       "ServiceAccount",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name:      "cloud-controller-manager",
-				Namespace: "kube-system",
-			},
-		},
-		ClusterRoleBinding: rbac.ClusterRoleBinding{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "rbac.authorization.k8s.io/v1",
-				Kind:       "ClusterRoleBinding",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name: "system:cloud-controller-manager",
-			},
-			RoleRef: rbac.RoleRef{
-				APIGroup: "rbac.authorization.k8s.io",
-				Kind:     "ClusterRole",
-				Name:     "cluster-admin",
-			},
-			Subjects: []rbac.Subject{
-				{
-					Kind:      "ServiceAccount",
-					Name:      "cloud-controller-manager",
-					Namespace: "kube-system",
-				},
-			},
-		},
-		DaemonSet: apps.DaemonSet{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "apps/v1",
-				Kind:       "DaemonSet",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Labels: map[string]string{
-					"k8s-app": "cloud-controller-manager",
-				},
-				Name:      "cloud-controller-manager",
-				Namespace: "kube-system",
-			},
-			Spec: apps.DaemonSetSpec{
-				Selector: &meta.LabelSelector{
-					MatchLabels: map[string]string{
-						"k8s-app": "cloud-controller-manager",
-					},
-				},
-				Template: k8s.PodTemplateSpec{
-					ObjectMeta: meta.ObjectMeta{
-						Labels: map[string]string{
-							"k8s-app": "cloud-controller-manager",
-						},
-					},
-					Spec: k8s.PodSpec{
-						ServiceAccountName: "cloud-controller-manager",
-						Containers: []k8s.Container{
-							{
-								Name:         "cloud-controller-manager",
-								Image:        image,
-								Command:      command,
-								VolumeMounts: volumeMounts,
-								Env:          env,
-							},
-						},
-						Volumes: volumes,
-						Tolerations: []k8s.Toleration{
-							{
-								Key:    "node.cloudprovider.kubernetes.io/uninitialized",
-								Value:  "true",
-								Effect: k8s.TaintEffectNoSchedule,
-							},
-							{
-								Key:    "node-role.kubernetes.io/master",
-								Effect: k8s.TaintEffectNoSchedule,
-							},
-							{
-								Key:      "node-role.kubernetes.io/control-plane",
-								Operator: k8s.TolerationOpExists,
-								Effect:   k8s.TaintEffectNoSchedule,
-							},
-							{
-								Key:    "node.kubernetes.io/not-ready",
-								Effect: k8s.TaintEffectNoSchedule,
-							},
-						},
-						NodeSelector: map[string]string{
-							"node-role.kubernetes.io/control-plane": "",
-						},
-					},
-				},
-			},
-		},
-	}
-}
-
-func (c *CloudControllerManagerDeployment) Marshal() ([]byte, error) {
-	return kubernetes.MarshalK8SResources(c)
-}

bootstrapper/internal/kubernetes/k8sapi/resources/cloud_controller_manager_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-package resources
-
-import (
-	"testing"
-
-	"github.com/edgelesssys/constellation/v2/internal/kubernetes"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	k8s "k8s.io/api/core/v1"
-)
-
-func TestCloudControllerMarshalUnmarshal(t *testing.T) {
-	require := require.New(t)
-	assert := assert.New(t)
-
-	cloudControllerManagerDepl := NewDefaultCloudControllerManagerDeployment("dummy-cloudprovider", "some-image:latest", "/dummy_path", "192.0.2.0/24", []string{}, []k8s.Volume{}, []k8s.VolumeMount{}, nil)
-	data, err := cloudControllerManagerDepl.Marshal()
-	require.NoError(err)
-
-	var recreated CloudControllerManagerDeployment
-	require.NoError(kubernetes.UnmarshalK8SResources(data, &recreated))
-	assert.Equal(cloudControllerManagerDepl, &recreated)
-}

bootstrapper/internal/kubernetes/kubernetes.go

@@ -11,6 +11,7 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"strconv"
@@ -18,9 +19,11 @@ import (
 
 	"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
 	"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
+	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
+	"github.com/edgelesssys/constellation/v2/internal/gcpshared"
 	"github.com/edgelesssys/constellation/v2/internal/logger"
 	"github.com/edgelesssys/constellation/v2/internal/role"
 	"github.com/edgelesssys/constellation/v2/internal/versions"
@@ -201,7 +204,10 @@ func (k *KubeWrapper) InitCluster(
 		return nil, fmt.Errorf("setting up konnectivity: %w", err)
 	}
 
-	extraVals := setupExtraVals(k.initialMeasurementsJSON, idKeyDigest, measurementSalt)
+	extraVals, err := k.setupExtraVals(ctx, k.initialMeasurementsJSON, idKeyDigest, measurementSalt, subnetworkPodCIDR, cloudServiceAccountURI)
+	if err != nil {
+		return nil, fmt.Errorf("setting up extraVals: %w", err)
+	}
 
 	if err = k.helmClient.InstallConstellationServices(ctx, helmReleases.ConstellationServices, extraVals); err != nil {
 		return nil, fmt.Errorf("installing constellation-services: %w", err)
@@ -211,9 +217,6 @@ func (k *KubeWrapper) InitCluster(
 		return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err)
 	}
 
-	if err := k.setupCCM(ctx, subnetworkPodCIDR, cloudServiceAccountURI, instance, k8sVersion); err != nil {
-		return nil, fmt.Errorf("setting up cloud controller manager: %w", err)
-	}
 	if err := k.setupCloudNodeManager(k8sVersion); err != nil {
 		return nil, fmt.Errorf("setting up cloud node manager: %w", err)
 	}
@@ -328,34 +331,6 @@ func (k *KubeWrapper) GetKubeconfig() ([]byte, error) {
 	return k.kubeconfigReader.ReadKubeconfig()
 }
 
-func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServiceAccountURI string, instance metadata.InstanceMetadata, k8sVersion versions.ValidK8sVersion) error {
-	if !k.cloudControllerManager.Supported() {
-		return nil
-	}
-	ccmConfigMaps, err := k.cloudControllerManager.ConfigMaps()
-	if err != nil {
-		return fmt.Errorf("defining ConfigMaps for CCM: %w", err)
-	}
-	ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance.ProviderID, cloudServiceAccountURI)
-	if err != nil {
-		return fmt.Errorf("defining Secrets for CCM: %w", err)
-	}
-	ccmImage, err := k.cloudControllerManager.Image(k8sVersion)
-	if err != nil {
-		return fmt.Errorf("defining Image for CCM: %w", err)
-	}
-
-	cloudControllerManagerConfiguration := resources.NewDefaultCloudControllerManagerDeployment(
-		k.cloudControllerManager.Name(), ccmImage, k.cloudControllerManager.Path(), subnetworkPodCIDR,
-		k.cloudControllerManager.ExtraArgs(), k.cloudControllerManager.Volumes(), k.cloudControllerManager.VolumeMounts(), k.cloudControllerManager.Env(),
-	)
-	if err := k.clusterUtil.SetupCloudControllerManager(k.client, cloudControllerManagerConfiguration, ccmConfigMaps, ccmSecrets); err != nil {
-		return fmt.Errorf("setting up cloud-controller-manager: %w", err)
-	}
-
-	return nil
-}
-
 func (k *KubeWrapper) setupCloudNodeManager(k8sVersion versions.ValidK8sVersion) error {
 	if !k.cloudNodeManager.Supported() {
 		return nil
@@ -498,12 +473,82 @@ func getIPAddr() (string, error) {
 
 // setupExtraVals create a helm values map for consumption by helm-install.
 // Will move to a more dedicated place once that place becomes apparent.
-func setupExtraVals(initialMeasurementsJSON []byte, idkeydigest []byte, measurementSalt []byte) map[string]any {
+func (k *KubeWrapper) setupExtraVals(ctx context.Context, initialMeasurementsJSON []byte, idkeydigest []byte, measurementSalt []byte, subnetworkCIDR string, cloudServiceAccountURI string) (map[string]any, error) {
-	return map[string]any{
+	extraVals := map[string]any{
 		"join-service": map[string]any{
 			"measurements":    string(initialMeasurementsJSON),
-			"idkeydigest":     hex.EncodeToString(idkeydigest),
 			"measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt),
 		},
+		"ccm": map[string]any{
+			"subnetworkCIDR": subnetworkCIDR,
+		},
 	}
+
+	instance, err := k.providerMetadata.Self(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("retrieving current instance: %w", err)
+	}
+
+	switch cloudprovider.FromString(k.cloudProvider) {
+	case cloudprovider.GCP:
+		{
+			uid, err := k.providerMetadata.UID(ctx)
+			if err != nil {
+				return nil, fmt.Errorf("getting uid: %w", err)
+			}
+
+			projectID, _, _, err := gcpshared.SplitProviderID(instance.ProviderID)
+			if err != nil {
+				return nil, fmt.Errorf("splitting providerID: %w", err)
+			}
+
+			serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(cloudServiceAccountURI)
+			if err != nil {
+				return nil, fmt.Errorf("getting service account key: %w", err)
+			}
+			rawKey, err := json.Marshal(serviceAccountKey)
+			if err != nil {
+				return nil, fmt.Errorf("marshaling service account key: %w", err)
+			}
+
+			ccmVals, ok := extraVals["ccm"].(map[string]any)
+			if !ok {
+				return nil, errors.New("invalid ccm values")
+			}
+			ccmVals["GCP"] = map[string]any{
+				"projectID":  projectID,
+				"uid":        uid,
+				"secretData": string(rawKey),
+			}
+		}
+	case cloudprovider.Azure:
+		{
+			// TODO: After refactoring the ProviderMetadata interface this section should be rewritten.
+			// Currently, we have to rely on the Secrets(..) method, as GetNetworkSecurityGroupName & GetLoadBalancerName
+			// rely on Azure specific API endpoints.
+			ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance.ProviderID, cloudServiceAccountURI)
+			if err != nil {
+				return nil, fmt.Errorf("creating ccm secret: %w", err)
+			}
+			if len(ccmSecrets) < 1 {
+				return nil, errors.New("missing secret")
+			}
+			rawConfig := ccmSecrets[0].Data["azure.json"]
+
+			ccmVals, ok := extraVals["ccm"].(map[string]any)
+			if !ok {
+				return nil, errors.New("invalid ccm values")
+			}
+			ccmVals["Azure"] = map[string]any{
+				"azureConfig": string(rawConfig),
+			}
+
+			joinVals, ok := extraVals["join-service"].(map[string]any)
+			if !ok {
+				return nil, errors.New("invalid join-service values")
+			}
+			joinVals["idkeydigest"] = hex.EncodeToString(idkeydigest)
+		}
+	}
+	return extraVals, nil
 }

bootstrapper/internal/kubernetes/kubernetes_test.go

@@ -191,7 +191,7 @@ func TestInitCluster(t *testing.T) {
 			wantErr:                true,
 			k8sVersion:             versions.Default,
 		},
-		"kubeadm init fails when setting up the join service": {
+		"kubeadm init fails when setting up constellation-services chart": {
 			clusterUtil: stubClusterUtil{},
 			helmClient:  stubHelmClient{servicesError: someErr},
 			kubeconfigReader: &stubKubeconfigReader{
@@ -204,18 +204,6 @@ func TestInitCluster(t *testing.T) {
 			wantErr:                true,
 			k8sVersion:             versions.Default,
 		},
-		"kubeadm init fails when setting the cloud contoller manager": {
-			clusterUtil: stubClusterUtil{setupCloudControllerManagerError: someErr},
-			kubeconfigReader: &stubKubeconfigReader{
-				Kubeconfig: []byte("someKubeconfig"),
-			},
-			providerMetadata:       &stubProviderMetadata{},
-			CloudControllerManager: &stubCloudControllerManager{SupportedResp: true},
-			CloudNodeManager:       &stubCloudNodeManager{},
-			ClusterAutoscaler:      &stubClusterAutoscaler{},
-			wantErr:                true,
-			k8sVersion:             versions.Default,
-		},
 		"kubeadm init fails when setting the cloud node manager": {
 			clusterUtil: stubClusterUtil{setupCloudNodeManagerError: someErr},
 			kubeconfigReader: &stubKubeconfigReader{
@@ -252,19 +240,6 @@ func TestInitCluster(t *testing.T) {
 			wantErr:                true,
 			k8sVersion:             versions.Default,
 		},
-		"kubeadm init fails when setting up the kms": {
-			clusterUtil: stubClusterUtil{},
-			helmClient:  stubHelmClient{servicesError: someErr},
-			kubeconfigReader: &stubKubeconfigReader{
-				Kubeconfig: []byte("someKubeconfig"),
-			},
-			providerMetadata:       &stubProviderMetadata{SupportedResp: false},
-			CloudControllerManager: &stubCloudControllerManager{},
-			CloudNodeManager:       &stubCloudNodeManager{SupportedResp: false},
-			ClusterAutoscaler:      &stubClusterAutoscaler{},
-			wantErr:                true,
-			k8sVersion:             versions.Default,
-		},
 		"kubeadm init fails when setting up konnectivity": {
 			clusterUtil: stubClusterUtil{setupKonnectivityError: someErr},
 			kubeconfigReader: &stubKubeconfigReader{

cli/internal/cmd/helmloader.go

@@ -1,21 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-package cmd
-
-import "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
-
-type helmLoader interface {
-	Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) ([]byte, error)
-}
-
-type stubHelmLoader struct {
-	loadErr error
-}
-
-func (d *stubHelmLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) ([]byte, error) {
-	return nil, d.loadErr
-}

cli/internal/cmd/init.go

@@ -126,7 +126,7 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator
 		return fmt.Errorf("parsing or generating master secret from file %s: %w", flags.masterSecretPath, err)
 	}
 
-	helmDeployments, err := helmLoader.Load(provider, flags.conformance, masterSecret.Key, masterSecret.Salt, getEnforcedPCRs(provider, config), getEnforceIDKeyDigest(provider, config))
+	helmDeployments, err := helmLoader.Load(provider, flags.conformance, masterSecret.Key, masterSecret.Salt, getEnforcedPCRs(provider, config), getEnforceIDKeyDigest(provider, config), k8sVersion)
 	if err != nil {
 		return fmt.Errorf("loading Helm charts: %w", err)
 	}
@@ -368,3 +368,7 @@ func getMarshaledServiceAccountURI(provider cloudprovider.Provider, config *conf
 type grpcDialer interface {
 	Dial(ctx context.Context, target string) (*grpc.ClientConn, error)
 }
+
+type helmLoader interface {
+	Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool, k8sVersion versions.ValidK8sVersion) ([]byte, error)
+}

cli/internal/cmd/init_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/grpc/testdialer"
 	"github.com/edgelesssys/constellation/v2/internal/license"
 	"github.com/edgelesssys/constellation/v2/internal/oid"
+	"github.com/edgelesssys/constellation/v2/internal/versions"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -503,3 +504,11 @@ func (c *stubLicenseClient) QuotaCheck(ctx context.Context, checkRequest license
 		Quota: 25,
 	}, nil
 }
+
+type stubHelmLoader struct {
+	loadErr error
+}
+
+func (d *stubHelmLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool, k8sVersion versions.ValidK8sVersion) ([]byte, error) {
+	return nil, d.loadErr
+}

cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml

@@ -7,5 +7,21 @@ version: 2.2.0-pre
 dependencies:
   - name: kms
     version: 2.2.0-pre
+    tags:
+      - Azure
+      - GCP
+      - AWS
+      - QEMU
   - name: join-service
     version: 2.2.0-pre
+    tags:
+      - Azure
+      - GCP
+      - AWS
+      - QEMU
+  - name: ccm
+    version: 2.2.0-pre
+    tags:
+      - Azure
+      - GCP
+      - AWS

cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: ccm
+description: A Helm chart to deploy the cloud controller manager.
+type: application
+version: 2.2.0-pre

cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/aws-daemonset.yaml

@@ -0,0 +1,63 @@
+{{ if eq .Values.csp "AWS" }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cloud-controller-manager
+  namespace: {{ .Release.Namespace }}
+  labels:
+    k8s-app: cloud-controller-manager
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cloud-controller-manager
+  template:
+    metadata:
+      labels:
+        k8s-app: cloud-controller-manager
+    spec:
+      containers:
+        - name: cloud-controller-manager
+          image: {{ .Values.AWS.image }}
+          command:
+          - /aws-cloud-controller-manager
+          - --cloud-provider=aws
+          - --leader-elect=true
+          - --cluster-cidr={{ .Values.subnetworkCIDR }}
+          - -v=2
+          volumeMounts:
+          - name: etckubernetes
+            mountPath: /etc/kubernetes
+            readOnly: true
+          - name: etcssl
+            mountPath: /etc/ssl
+            readOnly: true
+          - name: etcpki
+            mountPath: /etc/pki
+            readOnly: true
+          resources: {}
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      serviceAccountName: cloud-controller-manager
+      tolerations:
+      - effect: NoSchedule
+        key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - effect: NoSchedule
+        key: node.kubernetes.io/not-ready
+      volumes:
+      - name: etckubernetes
+        hostPath:
+          path: /etc/kubernetes
+      - name: etcssl
+        hostPath:
+          path: /etc/ssl
+      - name: etcpki
+        hostPath:
+          path: /etc/pki
+  updateStrategy: {}
+{{ end }}

cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/azure-daemonset.yaml

@@ -0,0 +1,73 @@
+{{ if eq .Values.csp "Azure" }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cloud-controller-manager
+  namespace: {{ .Release.Namespace}}
+  labels:
+    k8s-app: cloud-controller-manager
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cloud-controller-manager
+  template:
+    metadata:
+      labels:
+        k8s-app: cloud-controller-manager
+    spec:
+      containers:
+        - name: cloud-controller-manager
+          image: {{ .Values.Azure.image }}
+          command:
+          - cloud-controller-manager
+          - --cloud-provider=azure
+          - --leader-elect=true
+          - --cluster-cidr={{ .Values.subnetworkCIDR }}
+          - -v=2
+          - --controllers=*,-cloud-node
+          - --cloud-config=/etc/azure/azure.json
+          - --allocate-node-cidrs=false
+          - --configure-cloud-routes=true
+          volumeMounts:
+          - name: etckubernetes
+            mountPath: /etc/kubernetes
+            readOnly: true
+          - name: etcssl
+            mountPath: /etc/ssl
+            readOnly: true
+          - name: etcpki
+            mountPath: /etc/pki
+            readOnly: true
+          - name: azureconfig
+            mountPath: /etc/azure
+            readOnly: true
+          resources: {}
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      serviceAccountName: cloud-controller-manager
+      tolerations:
+      - effect: NoSchedule
+        key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - effect: NoSchedule
+        key: node.kubernetes.io/not-ready
+      volumes:
+      - name: etckubernetes
+        hostPath:
+          path: /etc/kubernetes
+      - name: etcssl
+        hostPath:
+          path: /etc/ssl
+      - name: etcpki
+        hostPath:
+          path: /etc/pki
+      - name: azureconfig
+        secret:
+          secretName: azureconfig
+  updateStrategy: {}
+{{ end }}

cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/azure-secret.yaml

@@ -0,0 +1,9 @@
+{{ if eq .Values.csp "Azure" }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: azureconfig
+  namespace: {{ .Release.Namespace }}
+data:
+  azure.json: {{ .Values.Azure.azureConfig | b64enc }}
+{{ end }}

cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/clusterrolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: cloud-controller-manager
+    namespace: {{ .Release.Namespace }}

cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-cm.yaml

@@ -0,0 +1,9 @@
+{{ if eq .Values.csp "GCP" }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: gceconf
+  namespace: {{ .Release.Namespace }}
+data:
+  gce.conf: "[global]\nproject-id = {{.Values.GCP.projectID }}\nuse-metadata-server = true\nnode-tags = constellation-{{ .Values.GCP.uid }}\n"
+{{ end }}

cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-daemonset.yaml

@@ -0,0 +1,84 @@
+{{ if eq .Values.csp "GCP" }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cloud-controller-manager
+  namespace: {{ .Release.Namespace }}
+  labels:
+    k8s-app: cloud-controller-manager
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cloud-controller-manager
+  template:
+    metadata:
+      labels:
+        k8s-app: cloud-controller-manager
+    spec:
+      containers:
+        - name: cloud-controller-manager
+          image: {{ .Values.GCP.image }}
+          command:
+          - /cloud-controller-manager
+          - --cloud-provider=gce
+          - --leader-elect=true
+          - --cluster-cidr={{ .Values.subnetworkCIDR }}
+          - -v=2
+          - --use-service-account-credentials
+          - --controllers=cloud-node,cloud-node-lifecycle,nodeipam,service,route
+          - --cloud-config=/etc/gce/gce.conf
+          - --cidr-allocator-type=CloudAllocator
+          - --allocate-node-cidrs=true
+          - --configure-cloud-routes=false
+          volumeMounts:
+            - mountPath: /etc/kubernetes
+              name: etckubernetes
+              readOnly: true
+            - mountPath: /etc/ssl
+              name: etcssl
+              readOnly: true
+            - mountPath: /etc/pki
+              name: etcpki
+              readOnly: true
+            - mountPath: /etc/gce
+              name: gceconf
+              readOnly: true
+            - mountPath: /var/secrets/google
+              name: gcekey
+              readOnly: true
+          resources: {}
+          env:
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: /var/secrets/google/key.json
+      serviceAccountName: cloud-controller-manager
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      tolerations:
+        - effect: NoSchedule
+          key: node.cloudprovider.kubernetes.io/uninitialized
+          value: "true"
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
+          operator: Exists
+        - effect: NoSchedule
+          key: node.kubernetes.io/not-ready
+      volumes:
+        - name: etckubernetes
+          hostPath:
+            path: /etc/kubernetes
+        - name: etcssl
+          hostPath:
+            path: /etc/ssl
+        - name: etcpki
+          hostPath:
+            path: /etc/pki
+        - name: gceconf
+          configMap:
+            name: gceconf
+        - name: gcekey
+          secret:
+            secretName: gcekey
+  updateStrategy: {}
+{{ end }}

cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-secret.yaml

@@ -0,0 +1,9 @@
+{{ if eq .Values.csp "GCP" }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gcekey
+  namespace: {{ .Release.Namespace }}
+data:
+  key.json: {{ .Values.GCP.secretData | b64enc }}
+{{ end }}

cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/serviceaccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: {{ .Release.Namespace }}

cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.schema.json

@@ -0,0 +1,105 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema#",
+    "properties": {
+        "csp": {
+            "description": "CSP to which the chart is deployed.",
+            "enum": ["Azure", "GCP", "AWS", "QEMU"]
+        },
+        "subnetworkCIDR": {
+            "description": "CIDR for the subnetwork of the cluster",
+            "type": "string",
+            "examples": ["192.0.2.0/24"],
+            "pattern": "[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}/[0-9]{1,2}"
+        },
+        "GCP": {
+            "description": "Config values required for deployment on GCP",
+            "type": "object",
+            "properties": {
+                "image": {
+                    "description": "Container image to use for the spawned pods.",
+                    "type": "string"
+                },
+                "projectID": {
+                    "description": "ID of the GCP project into which the cluster is deployed",
+                    "type": "string",
+                    "examples": ["demoproject-581925"]
+                },
+                "uid": {
+                    "description": "Unique identifier for the cluster",
+                    "type": "string"
+                },
+                "secretData": {
+                    "description": "GCP service account key as a json-string",
+                    "type": "string"
+                }
+            },
+            "required": [
+                "image",
+                "projectID",
+                "uid",
+                "secretData"
+            ]
+        },
+        "Azure": {
+            "description": "Config values required for deployment on Azure",
+            "type": "object",
+            "properties": {
+                "image": {
+                    "description": "Container image to use for the spawned pods.",
+                    "type": "string",
+                    "examples": ["mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:latest"]
+                },
+                "azureConfig": {
+                    "description": "Base64 encoded json string that hold required config parameters for Azure CCM.",
+                    "type": "string"
+                }
+            },
+            "required": [
+                "image",
+                "azureConfig"
+            ]
+        },
+        "AWS": {
+            "description": "Config values required for deployment on AWS",
+            "type": "object",
+            "properties": {
+                "image": {
+                    "description": "Container image to use for the spawned pods.",
+                    "type": "string"
+                }
+            },
+            "required": [
+                "image"
+            ]
+        }
+    },
+    "required": [
+        "csp",
+        "subnetworkCIDR"
+    ],
+    "allOf": [
+        {
+            "if": {
+                "properties": { "csp": { "const": "GCP" } },
+                "required": ["csp"]
+            },
+            "then": { "required": ["GCP"] }
+        },
+        {
+            "if": {
+                "properties": { "csp": { "const": "Azure" } },
+                "required": ["csp"]
+            },
+            "then": { "required": ["Azure"] }
+        },
+        {
+            "if": {
+                "properties": { "csp": { "const": "AWS" } },
+                "required": ["csp"]
+            },
+            "then": { "required": ["AWS"] }
+        }
+    ],
+    "title": "Values",
+    "type": "object"
+}

cli/internal/helm/charts/edgeless/constellation-services/values.yaml

@@ -9,3 +9,10 @@ global:
   k8sVersionCMName: k8s-version
   # Name of the ConfigMap that holds configs that should not be modified by the user.
   internalCMName: internal-config
+
+# Set one of the tags to true to indicate which CSP you are deploying to.
+tags:
+  Azure: false
+  GCP: false
+  AWS: false
+  QEMU: false

cli/internal/helm/loader.go

@@ -36,13 +36,13 @@ var HelmFS embed.FS
 
 type ChartLoader struct{}
 
-func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) ([]byte, error) {
+func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool, k8sVersion versions.ValidK8sVersion) ([]byte, error) {
 	ciliumRelease, err := i.loadCilium(csp, conformanceMode)
 	if err != nil {
 		return nil, err
 	}
 
-	conServicesRelease, err := i.loadConstellationServices(csp, masterSecret, salt, enforcedPCRs, enforceIDKeyDigest)
+	conServicesRelease, err := i.loadConstellationServices(csp, masterSecret, salt, enforcedPCRs, enforceIDKeyDigest, k8sVersion)
 	if err != nil {
 		return nil, err
 	}
@@ -93,7 +93,7 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo
 // loadConstellationServices loads the constellation-services chart from the embed.FS, marshals it into a helm-package .tgz and sets the values that can be set in the CLI.
 func (i *ChartLoader) loadConstellationServices(csp cloudprovider.Provider,
 	masterSecret []byte, salt []byte, enforcedPCRs []uint32,
-	enforceIDKeyDigest bool,
+	enforceIDKeyDigest bool, k8sVersion versions.ValidK8sVersion,
 ) (helm.Release, error) {
 	chart, err := loadChartsDir(HelmFS, "charts/edgeless/constellation-services")
 	if err != nil {
@@ -134,14 +134,64 @@ func (i *ChartLoader) loadConstellationServices(csp cloudprovider.Provider,
 			"image":        versions.JoinImage,
 			"namespace":    constants.ConstellationNamespace,
 		},
+		"ccm": map[string]interface{}{
+			"csp": csp,
+		},
 	}
 
-	if csp == cloudprovider.Azure {
+	switch csp {
-		joinServiceVals, ok := vals["join-service"].(map[string]any)
+	case cloudprovider.Azure:
-		if !ok {
+		{
-			return helm.Release{}, errors.New("invalid join-service values")
+			joinServiceVals, ok := vals["join-service"].(map[string]any)
+			if !ok {
+				return helm.Release{}, errors.New("invalid join-service values")
+			}
+			joinServiceVals["enforceIdKeyDigest"] = enforceIDKeyDigest
+
+			ccmVals, ok := vals["ccm"].(map[string]any)
+			if !ok {
+				return helm.Release{}, errors.New("invalid ccm values")
+			}
+			ccmVals["Azure"] = map[string]any{
+				"image": versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure,
+			}
+
+			vals["tags"] = map[string]any{
+				"Azure": true,
+			}
+		}
+	case cloudprovider.GCP:
+		{
+			ccmVals, ok := vals["ccm"].(map[string]any)
+			if !ok {
+				return helm.Release{}, errors.New("invalid ccm values")
+			}
+			ccmVals["GCP"] = map[string]any{
+				"image": versions.VersionConfigs[k8sVersion].CloudControllerManagerImageGCP,
+			}
+
+			vals["tags"] = map[string]any{
+				"GCP": true,
+			}
+		}
+	case cloudprovider.QEMU:
+		{
+			vals["tags"] = map[string]interface{}{
+				"QEMU": true,
+			}
+		}
+	case cloudprovider.AWS:
+		ccmVals, ok := vals["ccm"].(map[string]any)
+		if !ok {
+			return helm.Release{}, errors.New("invalid ccm values")
+		}
+		ccmVals["AWS"] = map[string]any{
+			"image": versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAWS,
+		}
+
+		vals["tags"] = map[string]any{
+			"AWS": true,
 		}
-		joinServiceVals["enforceIdKeyDigest"] = enforceIDKeyDigest
 	}
 
 	return helm.Release{Chart: chartRaw, Values: vals, ReleaseName: "constellation-services", Wait: true}, nil

cli/internal/helm/loader_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
 	"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
+	"github.com/edgelesssys/constellation/v2/internal/versions"
 	"github.com/stretchr/testify/assert"
 	"helm.sh/helm/v3/pkg/chart/loader"
 )
@@ -21,7 +22,7 @@ func TestLoad(t *testing.T) {
 	assert := assert.New(t)
 
 	chartLoader := ChartLoader{}
-	release, err := chartLoader.Load(cloudprovider.GCP, true, []byte("secret"), []byte("salt"), nil, false)
+	release, err := chartLoader.Load(cloudprovider.GCP, true, []byte("secret"), []byte("salt"), nil, false, versions.Default)
 	assert.NoError(err)
 
 	var helmReleases helm.Releases