AB#2504: Deploy join-service via helm (#358)

CMakeLists.txt

@@ -63,4 +63,7 @@ add_test(NAME integration-node-operator COMMAND make test WORKING_DIRECTORY ${CM
 add_test(NAME integration-csi COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/csi)
 add_test(NAME integration-dm COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/disk-mapper/internal)
 add_test(NAME integration-license COMMAND bash -c "go test -tags integration" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/internal/license)
-add_test(NAME helm-lint COMMAND bash -c "helm lint * --set kms.image='ghcr.io/edgelesssys/constellation/kms:latest' --set kms.salt='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' --set kms.masterSecret='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/cli/internal/helm/charts/edgeless/)
+add_test(NAME helm-lint COMMAND bash -c "helm lint * --set kms.image='ghcr.io/edgelesssys/constellation/kms:latest' --set join-service.csp='QEMU' \
+  --set join-service.enforcedPCRs='[]' --set join-service.image='ghcr.io/edgelesssys/constellation/join-service:latest' --set join-service.measurements='[]' \
+  --set join-service.measurementSalt='deadbeef' --set kms.salt='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' --set kms.masterSecret='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'"
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/cli/internal/helm/charts/edgeless/)

bootstrapper/internal/helm/client.go

@@ -53,7 +53,7 @@ func New(log *logger.Logger) (*Client, error) {
 }
 
 // InstallConstellationServices installs the constellation-services chart. In the future this chart should bundle all microservices.
-func (h *Client) InstallConstellationServices(ctx context.Context, release helm.Release) error {
+func (h *Client) InstallConstellationServices(ctx context.Context, release helm.Release, extraVals map[string]interface{}) error {
 	h.Namespace = constants.HelmNamespace
 	h.ReleaseName = release.ReleaseName
 	h.Wait = release.Wait
@@ -61,7 +61,7 @@ func (h *Client) InstallConstellationServices(ctx context.Context, release helm.
 	// update dependencies - unsure if necessary for local deps.
 	h.DependencyUpdate = true
 
-	// TODO: Possibly fetch metadata to extend values here.
+	mergedVals := mergeMaps(release.Values, extraVals)
 
 	reader := bytes.NewReader(release.Chart)
 	chart, err := loader.LoadArchive(reader)
@@ -69,7 +69,7 @@ func (h *Client) InstallConstellationServices(ctx context.Context, release helm.
 		return fmt.Errorf("helm load archive: %w", err)
 	}
 
-	_, err = h.RunWithContext(ctx, chart, release.Values)
+	_, err = h.RunWithContext(ctx, chart, mergedVals)
 	if err != nil {
 		return fmt.Errorf("helm install services: %w", err)
 	}
@@ -77,6 +77,27 @@ func (h *Client) InstallConstellationServices(ctx context.Context, release helm.
 	return nil
 }
 
+// mergeMaps returns a new map that is the merger of it's inputs.
+// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108.
+func mergeMaps(a, b map[string]interface{}) map[string]interface{} {
+	out := make(map[string]interface{}, len(a))
+	for k, v := range a {
+		out[k] = v
+	}
+	for k, v := range b {
+		if v, ok := v.(map[string]interface{}); ok {
+			if bv, ok := out[k]; ok {
+				if bv, ok := bv.(map[string]interface{}); ok {
+					out[k] = mergeMaps(bv, v)
+					continue
+				}
+			}
+		}
+		out[k] = v
+	}
+	return out
+}
+
 // InstallCilium sets up the cilium pod network.
 func (h *Client) InstallCilium(ctx context.Context, kubectl k8sapi.Client, release helm.Release, in k8sapi.SetupPodNetworkInput) error {
 	h.Namespace = constants.HelmNamespace

bootstrapper/internal/helm/client_test.go

@@ -0,0 +1,92 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package helm
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMe(t *testing.T) {
+	testCases := map[string]struct {
+		vals      map[string]interface{}
+		extraVals map[string]interface{}
+		expected  map[string]interface{}
+	}{
+		"equal": {
+			vals: map[string]interface{}{
+				"join-service": map[string]interface{}{
+					"key1": "foo",
+					"key2": "bar",
+				},
+			},
+			extraVals: map[string]interface{}{
+				"join-service": map[string]interface{}{
+					"extraKey1": "extraFoo",
+					"extraKey2": "extraBar",
+				},
+			},
+			expected: map[string]interface{}{
+				"join-service": map[string]interface{}{
+					"key1":      "foo",
+					"key2":      "bar",
+					"extraKey1": "extraFoo",
+					"extraKey2": "extraBar",
+				},
+			},
+		},
+		"missing join-service extraVals": {
+			vals: map[string]interface{}{
+				"join-service": map[string]interface{}{
+					"key1": "foo",
+					"key2": "bar",
+				},
+			},
+			extraVals: map[string]interface{}{
+				"extraKey1": "extraFoo",
+				"extraKey2": "extraBar",
+			},
+			expected: map[string]interface{}{
+				"join-service": map[string]interface{}{
+					"key1": "foo",
+					"key2": "bar",
+				},
+				"extraKey1": "extraFoo",
+				"extraKey2": "extraBar",
+			},
+		},
+		"missing join-service vals": {
+			vals: map[string]interface{}{
+				"key1": "foo",
+				"key2": "bar",
+			},
+			extraVals: map[string]interface{}{
+				"join-service": map[string]interface{}{
+					"extraKey1": "extraFoo",
+					"extraKey2": "extraBar",
+				},
+			},
+			expected: map[string]interface{}{
+				"key1": "foo",
+				"key2": "bar",
+				"join-service": map[string]interface{}{
+					"extraKey1": "extraFoo",
+					"extraKey2": "extraBar",
+				},
+			},
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			newVals := mergeMaps(tc.vals, tc.extraVals)
+			assert.Equal(tc.expected, newVals)
+		})
+	}
+}

bootstrapper/internal/kubernetes/k8sapi/constants.go

@@ -12,7 +12,6 @@ const (
 	binDir                  = "/run/state/bin"
 	kubeadmPath             = "/run/state/bin/kubeadm"
 	kubeletPath             = "/run/state/bin/kubelet"
-	kubectlPath             = "/run/state/bin/kubectl"
 	kubeletServiceEtcPath   = "/run/systemd/system/kubelet.service"
 	kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service"
 	kubeadmConfEtcPath      = "/run/systemd/system/kubelet.service.d/10-kubeadm.conf"

bootstrapper/internal/kubernetes/k8sapi/k8sutil.go

@@ -334,11 +334,6 @@ func (k *KubernetesUtil) SetupAutoscaling(kubectl Client, clusterAutoscalerConfi
 	return kubectl.Apply(clusterAutoscalerConfiguration, true)
 }
 
-// SetupJoinService deploys the Constellation node join service.
-func (k *KubernetesUtil) SetupJoinService(kubectl Client, joinServiceConfiguration kubernetes.Marshaler) error {
-	return kubectl.Apply(joinServiceConfiguration, true)
-}
-
 // SetupGCPGuestAgent deploys the GCP guest agent daemon set.
 func (k *KubernetesUtil) SetupGCPGuestAgent(kubectl Client, guestAgentDaemonset kubernetes.Marshaler) error {
 	return kubectl.Apply(guestAgentDaemonset, true)

bootstrapper/internal/kubernetes/k8sapi/resources/joinservice.go

@@ -1,277 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-package resources
-
-import (
-	"fmt"
-
-	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
-	"github.com/edgelesssys/constellation/v2/internal/constants"
-	"github.com/edgelesssys/constellation/v2/internal/kubernetes"
-	"github.com/edgelesssys/constellation/v2/internal/versions"
-	apps "k8s.io/api/apps/v1"
-	k8s "k8s.io/api/core/v1"
-	rbac "k8s.io/api/rbac/v1"
-	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
-)
-
-type JoinServiceDaemonset struct {
-	ClusterRole        rbac.ClusterRole
-	ClusterRoleBinding rbac.ClusterRoleBinding
-	ConfigMap          k8s.ConfigMap
-	DaemonSet          apps.DaemonSet
-	ServiceAccount     k8s.ServiceAccount
-	Service            k8s.Service
-}
-
-// NewJoinServiceDaemonset returns a daemonset for the join service.
-func NewJoinServiceDaemonset(csp, measurementsJSON, enforcedPCRsJSON, initialIDKeyDigest, enforceIDKeyDigest string, measurementSalt []byte) *JoinServiceDaemonset {
-	joinConfigData := map[string]string{
-		constants.MeasurementsFilename: measurementsJSON,
-		constants.EnforcedPCRsFilename: enforcedPCRsJSON,
-	}
-	if cloudprovider.FromString(csp) == cloudprovider.Azure {
-		joinConfigData[constants.EnforceIDKeyDigestFilename] = enforceIDKeyDigest
-		joinConfigData[constants.IDKeyDigestFilename] = initialIDKeyDigest
-	}
-
-	return &JoinServiceDaemonset{
-		ClusterRole: rbac.ClusterRole{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "rbac.authorization.k8s.io/v1",
-				Kind:       "ClusterRole",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name: "join-service",
-				Labels: map[string]string{
-					"k8s-app": "join-service",
-				},
-			},
-			Rules: []rbac.PolicyRule{
-				{
-					APIGroups: []string{""},
-					Resources: []string{"secrets"},
-					Verbs:     []string{"get", "list", "create", "update"},
-				},
-				{
-					APIGroups: []string{"rbac.authorization.k8s.io"},
-					Resources: []string{"roles", "rolebindings"},
-					Verbs:     []string{"create", "update"},
-				},
-			},
-		},
-		ClusterRoleBinding: rbac.ClusterRoleBinding{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "rbac.authorization.k8s.io/v1",
-				Kind:       "ClusterRoleBinding",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name: "join-service",
-			},
-			RoleRef: rbac.RoleRef{
-				APIGroup: "rbac.authorization.k8s.io",
-				Kind:     "ClusterRole",
-				Name:     "join-service",
-			},
-			Subjects: []rbac.Subject{
-				{
-					Kind:      "ServiceAccount",
-					Name:      "join-service",
-					Namespace: constants.ConstellationNamespace,
-				},
-			},
-		},
-		DaemonSet: apps.DaemonSet{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "apps/v1",
-				Kind:       "DaemonSet",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name:      "join-service",
-				Namespace: constants.ConstellationNamespace,
-				Labels: map[string]string{
-					"k8s-app":                       "join-service",
-					"component":                     "join-service",
-					"kubernetes.io/cluster-service": "true",
-				},
-			},
-			Spec: apps.DaemonSetSpec{
-				Selector: &meta.LabelSelector{
-					MatchLabels: map[string]string{
-						"k8s-app": "join-service",
-					},
-				},
-				Template: k8s.PodTemplateSpec{
-					ObjectMeta: meta.ObjectMeta{
-						Labels: map[string]string{
-							"k8s-app": "join-service",
-						},
-					},
-					Spec: k8s.PodSpec{
-						PriorityClassName:  "system-cluster-critical",
-						ServiceAccountName: "join-service",
-						Tolerations: []k8s.Toleration{
-							{
-								Key:      "CriticalAddonsOnly",
-								Operator: k8s.TolerationOpExists,
-							},
-							{
-								Key:      "node-role.kubernetes.io/master",
-								Operator: k8s.TolerationOpEqual,
-								Value:    "true",
-								Effect:   k8s.TaintEffectNoSchedule,
-							},
-							{
-								Key:      "node-role.kubernetes.io/control-plane",
-								Operator: k8s.TolerationOpExists,
-								Effect:   k8s.TaintEffectNoSchedule,
-							},
-							{
-								Operator: k8s.TolerationOpExists,
-								Effect:   k8s.TaintEffectNoExecute,
-							},
-							{
-								Operator: k8s.TolerationOpExists,
-								Effect:   k8s.TaintEffectNoSchedule,
-							},
-						},
-						// Only run on control plane nodes
-						NodeSelector: map[string]string{
-							"node-role.kubernetes.io/control-plane": "",
-						},
-						Containers: []k8s.Container{
-							{
-								Name:  "join-service",
-								Image: versions.JoinImage,
-								Ports: []k8s.ContainerPort{
-									{
-										ContainerPort: constants.JoinServicePort,
-										Name:          "tcp",
-									},
-								},
-								SecurityContext: &k8s.SecurityContext{
-									Privileged: func(b bool) *bool { return &b }(true),
-								},
-								Args: []string{
-									fmt.Sprintf("--cloud-provider=%s", csp),
-									fmt.Sprintf("--kms-endpoint=kms.kube-system:%d", constants.KMSPort),
-								},
-								VolumeMounts: []k8s.VolumeMount{
-									{
-										Name:      "config",
-										ReadOnly:  true,
-										MountPath: constants.ServiceBasePath,
-									},
-									{
-										Name:      "kubeadm",
-										ReadOnly:  true,
-										MountPath: "/etc/kubernetes",
-									},
-								},
-							},
-						},
-						Volumes: []k8s.Volume{
-							{
-								Name: "config",
-								VolumeSource: k8s.VolumeSource{
-									Projected: &k8s.ProjectedVolumeSource{
-										Sources: []k8s.VolumeProjection{
-											{
-												ConfigMap: &k8s.ConfigMapProjection{
-													LocalObjectReference: k8s.LocalObjectReference{
-														Name: constants.JoinConfigMap,
-													},
-												},
-											},
-											{
-												ConfigMap: &k8s.ConfigMapProjection{
-													LocalObjectReference: k8s.LocalObjectReference{
-														Name: constants.K8sVersion,
-													},
-												},
-											},
-											{
-												ConfigMap: &k8s.ConfigMapProjection{
-													LocalObjectReference: k8s.LocalObjectReference{
-														Name: constants.InternalConfigMap,
-													},
-												},
-											},
-										},
-									},
-								},
-							},
-							{
-								Name: "kubeadm",
-								VolumeSource: k8s.VolumeSource{
-									HostPath: &k8s.HostPathVolumeSource{
-										Path: "/etc/kubernetes",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-		ServiceAccount: k8s.ServiceAccount{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "v1",
-				Kind:       "ServiceAccount",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name:      "join-service",
-				Namespace: constants.ConstellationNamespace,
-			},
-		},
-		Service: k8s.Service{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "v1",
-				Kind:       "Service",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name:      "join-service",
-				Namespace: constants.ConstellationNamespace,
-			},
-			Spec: k8s.ServiceSpec{
-				Type: k8s.ServiceTypeNodePort,
-				Ports: []k8s.ServicePort{
-					{
-						Name:       "grpc",
-						Protocol:   k8s.ProtocolTCP,
-						Port:       constants.JoinServicePort,
-						TargetPort: intstr.IntOrString{IntVal: constants.JoinServicePort},
-						NodePort:   constants.JoinServiceNodePort,
-					},
-				},
-				Selector: map[string]string{
-					"k8s-app": "join-service",
-				},
-			},
-		},
-		ConfigMap: k8s.ConfigMap{
-			TypeMeta: meta.TypeMeta{
-				APIVersion: "v1",
-				Kind:       "ConfigMap",
-			},
-			ObjectMeta: meta.ObjectMeta{
-				Name:      constants.JoinConfigMap,
-				Namespace: constants.ConstellationNamespace,
-			},
-			Data: joinConfigData,
-			BinaryData: map[string][]byte{
-				constants.MeasurementSaltFilename: measurementSalt,
-			},
-		},
-	}
-}
-
-// Marshal the daemonset using the Kubernetes resource marshaller.
-func (a *JoinServiceDaemonset) Marshal() ([]byte, error) {
-	return kubernetes.MarshalK8SResources(a)
-}

bootstrapper/internal/kubernetes/k8sapi/resources/joinservice_test.go

@@ -1,25 +0,0 @@
-/*
-Copyright (c) Edgeless Systems GmbH
-
-SPDX-License-Identifier: AGPL-3.0-only
-*/
-
-package resources
-
-import (
-	"testing"
-
-	"github.com/edgelesssys/constellation/v2/internal/kubernetes"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestNewJoinServiceDaemonset(t *testing.T) {
-	deployment := NewJoinServiceDaemonset("csp", "measurementsJSON", "enforcedPCRsJSON", "deadbeef", "true", []byte{0x0, 0x1, 0x2})
-	deploymentYAML, err := deployment.Marshal()
-	require.NoError(t, err)
-
-	var recreated JoinServiceDaemonset
-	require.NoError(t, kubernetes.UnmarshalK8SResources(deploymentYAML, &recreated))
-	assert.Equal(t, deployment, &recreated)
-}

bootstrapper/internal/kubernetes/k8sutil.go

@@ -24,7 +24,6 @@ type clusterUtil interface {
 	JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error
 	SetupAccessManager(kubectl k8sapi.Client, sshUsers kubernetes.Marshaler) error
 	SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration kubernetes.Marshaler, secrets kubernetes.Marshaler) error
-	SetupJoinService(kubectl k8sapi.Client, joinServiceConfiguration kubernetes.Marshaler) error
 	SetupCloudControllerManager(kubectl k8sapi.Client, cloudControllerManagerConfiguration kubernetes.Marshaler, configMaps kubernetes.Marshaler, secrets kubernetes.Marshaler) error
 	SetupCloudNodeManager(kubectl k8sapi.Client, cloudNodeManagerConfiguration kubernetes.Marshaler) error
 	SetupKonnectivity(kubectl k8sapi.Client, konnectivityAgentsDaemonSet kubernetes.Marshaler) error
@@ -42,5 +41,5 @@ type clusterUtil interface {
 // Naming is inspired by Helm.
 type helmClient interface {
 	InstallCilium(context.Context, k8sapi.Client, helm.Release, k8sapi.SetupPodNetworkInput) error
-	InstallConstellationServices(ctx context.Context, release helm.Release) error
+	InstallConstellationServices(ctx context.Context, release helm.Release, extraVals map[string]interface{}) error
 }

bootstrapper/internal/kubernetes/kubernetes.go

@@ -8,6 +8,7 @@ package kubernetes
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -200,18 +201,16 @@ func (k *KubeWrapper) InitCluster(
 		return nil, fmt.Errorf("setting up konnectivity: %w", err)
 	}
 
-	if err = k.helmClient.InstallConstellationServices(ctx, helmReleases.ConstellationServices); err != nil {
+	extraVals := setupExtraVals(k.initialMeasurementsJSON, idKeyDigest, measurementSalt)
-		return nil, fmt.Errorf("installing kms: %w", err)
+
+	if err = k.helmClient.InstallConstellationServices(ctx, helmReleases.ConstellationServices, extraVals); err != nil {
+		return nil, fmt.Errorf("installing constellation-services: %w", err)
 	}
 
 	if err := k.setupInternalConfigMap(ctx, strconv.FormatBool(azureCVM)); err != nil {
 		return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err)
 	}
 
-	if err := k.setupJoinService(k.cloudProvider, k.initialMeasurementsJSON, measurementSalt, enforcedPCRs, idKeyDigest, enforceIDKeyDigest); err != nil {
-		return nil, fmt.Errorf("setting up join service failed: %w", err)
-	}
-
 	if err := k.setupCCM(ctx, subnetworkPodCIDR, cloudServiceAccountURI, instance, k8sVersion); err != nil {
 		return nil, fmt.Errorf("setting up cloud controller manager: %w", err)
 	}
@@ -329,21 +328,6 @@ func (k *KubeWrapper) GetKubeconfig() ([]byte, error) {
 	return k.kubeconfigReader.ReadKubeconfig()
 }
 
-func (k *KubeWrapper) setupJoinService(
-	csp string, measurementsJSON, measurementSalt []byte, enforcedPCRs []uint32, initialIDKeyDigest []byte, enforceIDKeyDigest bool,
-) error {
-	enforcedPCRsJSON, err := json.Marshal(enforcedPCRs)
-	if err != nil {
-		return fmt.Errorf("marshaling enforcedPCRs: %w", err)
-	}
-
-	joinConfiguration := resources.NewJoinServiceDaemonset(
-		csp, string(measurementsJSON), string(enforcedPCRsJSON), hex.EncodeToString(initialIDKeyDigest), strconv.FormatBool(enforceIDKeyDigest), measurementSalt,
-	)
-
-	return k.clusterUtil.SetupJoinService(k.client, joinConfiguration)
-}
-
 func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServiceAccountURI string, instance metadata.InstanceMetadata, k8sVersion versions.ValidK8sVersion) error {
 	if !k.cloudControllerManager.Supported() {
 		return nil
@@ -511,3 +495,15 @@ func getIPAddr() (string, error) {
 
 	return localAddr.IP.String(), nil
 }
+
+// setupExtraVals create a helm values map for consumption by helm-install.
+// Will move to a more dedicated place once that place becomes apparent.
+func setupExtraVals(initialMeasurementsJSON []byte, idkeydigest []byte, measurementSalt []byte) map[string]interface{} {
+	return map[string]interface{}{
+		"join-service": map[string]interface{}{
+			"measurements":    string(initialMeasurementsJSON),
+			"idkeydigest":     hex.EncodeToString(idkeydigest),
+			"measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt),
+		},
+	}
+}

bootstrapper/internal/kubernetes/kubernetes_test.go

@@ -192,7 +192,8 @@ func TestInitCluster(t *testing.T) {
 			k8sVersion:             versions.Default,
 		},
 		"kubeadm init fails when setting up the join service": {
-			clusterUtil: stubClusterUtil{setupJoinServiceError: someErr},
+			clusterUtil: stubClusterUtil{},
+			helmClient:  stubHelmClient{servicesError: someErr},
 			kubeconfigReader: &stubKubeconfigReader{
 				Kubeconfig: []byte("someKubeconfig"),
 			},
@@ -532,7 +533,6 @@ type stubClusterUtil struct {
 	installComponentsErr             error
 	initClusterErr                   error
 	setupAutoscalingError            error
-	setupJoinServiceError            error
 	setupCloudControllerManagerError error
 	setupCloudNodeManagerError       error
 	setupKonnectivityError           error
@@ -566,10 +566,6 @@ func (s *stubClusterUtil) SetupAutoscaling(kubectl k8sapi.Client, clusterAutosca
 	return s.setupAutoscalingError
 }
 
-func (s *stubClusterUtil) SetupJoinService(kubectl k8sapi.Client, joinServiceConfiguration kubernetes.Marshaler) error {
-	return s.setupJoinServiceError
-}
-
 func (s *stubClusterUtil) SetupGCPGuestAgent(kubectl k8sapi.Client, gcpGuestAgentConfiguration kubernetes.Marshaler) error {
 	return s.setupGCPGuestAgentErr
 }
@@ -688,6 +684,6 @@ func (s *stubHelmClient) InstallCilium(ctx context.Context, kubectl k8sapi.Clien
 	return s.ciliumError
 }
 
-func (s *stubHelmClient) InstallConstellationServices(ctx context.Context, release helm.Release) error {
+func (s *stubHelmClient) InstallConstellationServices(ctx context.Context, release helm.Release, extraVals map[string]interface{}) error {
 	return s.servicesError
 }

cli/internal/cmd/helmloader.go

@@ -9,13 +9,13 @@ package cmd
 import "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
 
 type helmLoader interface {
-	Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte) ([]byte, error)
+	Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) ([]byte, error)
 }
 
 type stubHelmLoader struct {
 	loadErr error
 }
 
-func (d *stubHelmLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte) ([]byte, error) {
+func (d *stubHelmLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) ([]byte, error) {
 	return nil, d.loadErr
 }

cli/internal/cmd/init.go

@@ -126,7 +126,7 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator
 		return fmt.Errorf("parsing or generating master secret from file %s: %w", flags.masterSecretPath, err)
 	}
 
-	helmDeployments, err := helmLoader.Load(provider, flags.conformance, masterSecret.Key, masterSecret.Salt)
+	helmDeployments, err := helmLoader.Load(provider, flags.conformance, masterSecret.Key, masterSecret.Salt, getEnforcedPCRs(provider, config), getEnforceIDKeyDigest(provider, config))
 	if err != nil {
 		return fmt.Errorf("loading Helm charts: %w", err)
 	}
@@ -143,7 +143,7 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator
 		KubernetesVersion:      config.KubernetesVersion,
 		SshUserKeys:            ssh.ToProtoSlice(sshUsers),
 		HelmDeployments:        helmDeployments,
-		EnforcedPcrs:           getEnforcedMeasurements(provider, config),
+		EnforcedPcrs:           getEnforcedPCRs(provider, config),
 		EnforceIdkeydigest:     getEnforceIDKeyDigest(provider, config),
 		ConformanceMode:        flags.conformance,
 	}
@@ -229,7 +229,7 @@ func writeRow(wr io.Writer, col1 string, col2 string) {
 	fmt.Fprint(wr, col1, "\t", col2, "\n")
 }
 
-func getEnforcedMeasurements(provider cloudprovider.Provider, config *config.Config) []uint32 {
+func getEnforcedPCRs(provider cloudprovider.Provider, config *config.Config) []uint32 {
 	switch provider {
 	case cloudprovider.Azure:
 		return config.Provider.Azure.EnforcedMeasurements

cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml

@@ -7,3 +7,5 @@ version: 2.2.0-pre
 dependencies:
   - name: kms
     version: 2.2.0-pre
+  - name: join-service
+    version: 2.2.0-pre

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: join-service
+description: A chart to deploy the Constellation join-service
+type: application
+version: 2.2.0-pre

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/clusterrole.yaml

@@ -0,0 +1,24 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-app: join-service
+  name: join-service
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - create
+  - update
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  - rolebindings
+  verbs:
+  - create
+  - update

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/clusterrolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: join-service
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: join-service
+subjects:
+- kind: ServiceAccount
+  name: join-service
+  namespace: {{ .Values.namespace }}

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/configmap.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: join-config
+  namespace: kube-system
+data:
+  # mustToJson is required so the json-strings passed from go are properly quoted in the rendered yaml.
+  enforcedPCRs: {{ .Values.enforcedPCRs | mustToJson }}
+  measurements: {{ .Values.measurements | mustToJson }}
+  {{- if eq .Values.csp "azure" }}
+  enforceIdKeyDigest: {{ .Values.enforceIdKeyDigest }}
+  idkeydigest: {{ .Values.idkeydigest }}
+  {{- end }}
+binaryData:
+  measurementSalt: {{ .Values.measurementSalt }}

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml

@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: join-service
+  namespace: {{ .Values.namespace }}
+  labels:
+    component: join-service
+    k8s-app: join-service
+    kubernetes.io/cluster-service: "true"
+spec:
+  selector:
+    matchLabels:
+      k8s-app: join-service
+  template:
+    metadata:
+      labels:
+        k8s-app: join-service
+    spec:
+      priorityClassName: system-cluster-critical
+      serviceAccountName: join-service
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/master
+          operator: Equal
+          value: "true"
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+        - effect: NoSchedule
+          operator: Exists
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      containers:
+        - name: join-service
+          image: {{ .Values.image }}
+          args:
+            - --cloud-provider={{ .Values.csp }}
+            - --kms-endpoint=kms.kube-system:{{ .Values.global.kmsPort }}
+          volumeMounts:
+            - mountPath: {{ .Values.global.serviceBasePath }}
+              name: config
+              readOnly: true
+            - mountPath: /etc/kubernetes
+              name: kubeadm
+              readOnly: true
+          ports:
+            - containerPort: {{ .Values.joinServicePort }}
+              name: tcp
+          resources: {}
+          securityContext:
+            privileged: true
+      volumes:
+        - name: config
+          projected:
+            sources:
+              - configMap:
+                  name: {{ .Values.global.joinConfigCMName }}
+              - configMap:
+                  name: {{ .Values.global.k8sVersionCMName }}
+              - configMap:
+                  name: {{ .Values.global.internalCMName }}
+        - name: kubeadm
+          hostPath:
+            path: /etc/kubernetes
+  updateStrategy: {}

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: join-service
+  namespace: {{ .Values.namespace }}
+spec:
+  type: NodePort
+  selector:
+    k8s-app: join-service
+  ports:
+  - name: grpc
+    protocol: TCP
+    port: {{ .Values.joinServicePort }}
+    targetPort: {{ .Values.joinServicePort }}
+    nodePort: {{ .Values.joinServiceNodePort }}
+status:
+  loadBalancer: {}

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/templates/serviceaccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: join-service
+  namespace: {{ .Values.namespace }}

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/values.schema.json

@@ -0,0 +1,53 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema#",
+    "properties": {
+        "csp": {
+            "description": "CSP to which the chart is deployed.",
+            "enum": ["Azure", "GCP", "AWS", "QEMU"]
+        },
+        "enforcedPCRs": {
+            "description": "JSON-string to describe the enforced PCRs.",
+            "type": "string",
+            "examples": ["[1, 15]"]
+        },
+        "measurements": {
+            "description": "JSON-string to describe the expected measurements.",
+            "type": "string",
+            "examples": ["{'1':'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA','15':'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='}"]
+        },
+        "enforceIdKeyDigest": {
+            "description": "Whether or not idkeydigest should be enforced during attestation on azure.",
+            "type": "boolean"
+        },
+        "idkeydigest": {
+            "description": "Expected idkeydigest value for Azure SNP attestation.",
+            "type": "string",
+            "examples": ["57486a447ec0f1958002a22a06b7673b9fd27d11e1c6527498056054c5fa92d23c50f9de44072760fe2b6fb89740b696"]
+        },
+        "image": {
+            "description": "Container image to use for the spawned pods.",
+            "type": "string",
+            "examples": ["ghcr.io/edgelesssys/constellation/join-service:latest"],
+            "pattern": "ghcr.io/edgelesssys/constellation/join-service:.+"
+        },
+        "measurementSalt": {
+            "description": "Salt used to generate node measurements",
+            "type": "string",
+            "examples": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"]
+        }
+    },
+    "required": [
+        "csp",
+        "enforcedPCRs",
+        "measurements",
+        "measurementSalt",
+        "image"
+    ],
+    "if": {
+        "properties": { "csp": { "const": "azure" } },
+        "required": ["csp"]
+    },
+    "then": { "required": ["enforceIdKeyDigest", "idkeydigest"] },
+    "title": "Values",
+    "type": "object"
+}

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/values.yaml

@@ -0,0 +1,5 @@
+# Namespace to which to deploy
+namespace: "kube-system"
+csp: "gcp"
+joinServicePort: 9090
+joinServiceNodePort: 30090

cli/internal/helm/charts/edgeless/constellation-services/charts/kms/templates/daemonset.yaml

@@ -17,15 +17,15 @@ spec:
         k8s-app: kms
     spec:
       containers:
-      - args:
+        - name: kms
-        - --port={{ .Values.port }}
           image: {{ .Values.image }}
-        name: kms
+          args:
-        resources: {}
+            - --port={{ .Values.global.kmsPort }}
           volumeMounts:
-        - mountPath: {{ .Values.serviceBasePath }}
+            - mountPath: {{ .Values.global.serviceBasePath }}
               name: config
               readOnly: true
+          resources: {}
       nodeSelector:
         node-role.kubernetes.io/control-plane: ""
       priorityClassName: system-cluster-critical
@@ -52,7 +52,7 @@ spec:
               items:
               - key: {{ .Values.measurementsFilename }}
                 path: {{ .Values.measurementsFilename }}
-              name: {{ .Values.joinConfigCMName }}
+              name: {{ .Values.global.joinConfigCMName }}
           - secret:
               items:
               - key: {{ .Values.masterSecretKeyName }}

cli/internal/helm/charts/edgeless/constellation-services/charts/kms/templates/service.yaml

@@ -6,9 +6,9 @@ metadata:
 spec:
   ports:
   - name: grpc
-    port: {{ .Values.port }}
+    port: {{ .Values.global.kmsPort }}
     protocol: TCP
-    targetPort: {{ .Values.port }}
+    targetPort: {{ .Values.global.kmsPort }}
   selector:
     k8s-app: kms
   type: ClusterIP

cli/internal/helm/charts/edgeless/constellation-services/charts/kms/values.yaml

@@ -1,11 +1,5 @@
 # Namespace to which KMS will be deployed.
 namespace: "kube-system"
-# Port on which the service will listen.
-port: 9000
-# Name of the ConfigMap that holds measurements and other info.
-joinConfigCMName: join-config
-# Path to which secrets/CMs are mounted.
-serviceBasePath: /var/config
 # Name of the key within the respective secret that holds the salt.
 saltKeyName: salt
 # Name of the secret that contains the master secret.

cli/internal/helm/charts/edgeless/constellation-services/values.yaml

@@ -0,0 +1,11 @@
+global:
+  # Port on which the KMS service will listen. Global since join-service also uses the value.
+  kmsPort: 9000
+  # Path to which secrets/CMs are mounted.
+  serviceBasePath: /var/config
+  # Name of the ConfigMap that holds measurements and other info.
+  joinConfigCMName: join-config
+  # Name of the ConfigMap that holds the installed k8s version.
+  k8sVersionCMName: k8s-version
+  # Name of the ConfigMap that holds configs that should not be modified by the user.
+  internalCMName: internal-config

cli/internal/helm/loader.go

@@ -36,13 +36,13 @@ var HelmFS embed.FS
 
 type ChartLoader struct{}
 
-func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte) ([]byte, error) {
+func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) ([]byte, error) {
 	ciliumRelease, err := i.loadCilium(csp, conformanceMode)
 	if err != nil {
 		return nil, err
 	}
 
-	conServicesRelease, err := i.loadConstellationServices(masterSecret, salt)
+	conServicesRelease, err := i.loadConstellationServices(csp, masterSecret, salt, enforcedPCRs, enforceIDKeyDigest)
 	if err != nil {
 		return nil, err
 	}
@@ -90,7 +90,8 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo
 	return helm.Release{Chart: chartRaw, Values: ciliumVals, ReleaseName: "cilium", Wait: true}, nil
 }
 
-func (i *ChartLoader) loadConstellationServices(masterSecret []byte, salt []byte) (helm.Release, error) {
+// loadConstellationServices loads the constellation-services chart from the embed.FS, marshals it into a helm-package .tgz and sets the values that can be set in the CLI.
+func (i *ChartLoader) loadConstellationServices(csp cloudprovider.Provider, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) (helm.Release, error) {
 	chart, err := loadChartsDir(HelmFS, "charts/edgeless/constellation-services")
 	if err != nil {
 		return helm.Release{}, fmt.Errorf("loading constellation-services chart: %w", err)
@@ -101,20 +102,40 @@ func (i *ChartLoader) loadConstellationServices(masterSecret []byte, salt []byte
 		return helm.Release{}, fmt.Errorf("packaging chart: %w", err)
 	}
 
+	enforcedPCRsJSON, err := json.Marshal(enforcedPCRs)
+	if err != nil {
+		return helm.Release{}, fmt.Errorf("marshaling enforcedPCRs: %w", err)
+	}
+
 	vals := map[string]interface{}{
-		"kms": map[string]interface{}{
+		"global": map[string]interface{}{
-			"namespace":            constants.ConstellationNamespace,
+			"kmsPort":          constants.KMSPort,
-			"port":                 constants.KMSPort,
-			"joinConfigCMName":     constants.JoinConfigMap,
 			"serviceBasePath":  constants.ServiceBasePath,
+			"joinConfigCMName": constants.JoinConfigMap,
+			"k8sVersionCMName": constants.K8sVersion,
+			"internalCMName":   constants.InternalConfigMap,
+		},
+		"kms": map[string]interface{}{
 			"image":                versions.KmsImage,
-			"masterSecretName":     constants.ConstellationMasterSecretStoreName,
-			"masterSecretKeyName":  constants.ConstellationMasterSecretKey,
-			"saltKeyName":          constants.ConstellationSaltKey,
-			"measurementsFilename": constants.MeasurementsFilename,
 			"masterSecret":         base64.StdEncoding.EncodeToString(masterSecret),
 			"salt":                 base64.StdEncoding.EncodeToString(salt),
+			"namespace":            constants.ConstellationNamespace,
+			"saltKeyName":          constants.ConstellationSaltKey,
+			"masterSecretKeyName":  constants.ConstellationMasterSecretKey,
+			"masterSecretName":     constants.ConstellationMasterSecretStoreName,
+			"measurementsFilename": constants.MeasurementsFilename,
 		},
+		"join-service": map[string]interface{}{
+			"csp":          csp,
+			"enforcedPCRs": string(enforcedPCRsJSON),
+			"image":        versions.JoinImage,
+			"namespace":    constants.ConstellationNamespace,
+		},
+	}
+
+	if csp == cloudprovider.Azure {
+		joinServiceVals := vals["join-service"].(map[string]interface{})
+		joinServiceVals["enforceIDKeyDigest"] = enforceIDKeyDigest
 	}
 
 	return helm.Release{Chart: chartRaw, Values: vals, ReleaseName: "constellation-services", Wait: true}, nil

cli/internal/helm/loader_test.go

@@ -21,7 +21,7 @@ func TestLoad(t *testing.T) {
 	assert := assert.New(t)
 
 	chartLoader := ChartLoader{}
-	release, err := chartLoader.Load(cloudprovider.GCP, true, []byte("secret"), []byte("salt"))
+	release, err := chartLoader.Load(cloudprovider.GCP, true, []byte("secret"), []byte("salt"), nil, false)
 	assert.NoError(err)
 
 	var helmReleases helm.Releases

image/README.md

@@ -32,7 +32,8 @@
         jq \
         util-linux \
         virt-manager \
-        python3-crc32c
+        python3-crc32c \
+        rpm
     ```
 
     </details>